From c253f4930e64cc15e3f866f4e7028bcab5b64d2d Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 13 Aug 2026 18:54:25 +0000 Subject: [PATCH 01/20] Document Query Store diagnostics baseline Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 487 +++++++++++++++++++++++ 1 file changed, 487 insertions(+) create mode 100644 docs/QueryStorePerformanceDiagnostics.md diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md new file mode 100644 index 0000000000..2efdb49377 --- /dev/null +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -0,0 +1,487 @@ +# Query Store Performance Diagnostics - Baseline Specification + +## Status + +Agreed baseline for implementation. This document defines the SQL contract, security boundary, operational limits, and validation requirements. It does not select a Geneva action, PaaS API, or direct-SQL caller. + +## Problem + +FHIR Azure SQL performance investigations currently require privileged, manual access to Query Store plans, runtime metrics, wait statistics, and statistics metadata. Existing Log Analytics data provides some Query Store information, but support engineers still need a bounded way to: + +- identify expensive or regressed query plans; +- retrieve an SSMS-viewable Query Store Showplan; +- compare runtime and wait metrics; and +- inspect statistics freshness, sampling, and cardinality metadata. + +The baseline is a self-contained, read-only SQL interface. Filtering, validation, redaction, paging, permissions, and auditing must live in SQL so the procedures can be used by an authorized direct SQL connection or wrapped by future operational tooling. + +Related internal guidance: + +- `Health.wiki/Home/Olympus-Team/DRI/TSGs/SQL-Latency-Issues.md` +- `Health.wiki/Home/Olympus-Team/Knowledge-Base/Datastore/SQL-DB/SQL-Query-Store---help-with-analyzing-queries.md` +- `Health.wiki/Home/Olympus-Team/DRI/TSGs/SQL-Latency-Issues/SQL-Statistics-Overview.md` +- `Health.wiki/Home/Olympus-Team/Development/SQL-Performance-Automation.md` + +## Goals + +1. Identify slow or resource-intensive query plans over a bounded time range. +2. Return full Query Store query text to authorized diagnostic callers. +3. Return an SSMS-viewable Query Store Showplan after removing parameter-value metadata. +4. Include Query Store wait statistics in slow-query results when capture is available. +5. Report statistics freshness, sampling, and filter metadata without returning histogram values. +6. Provide an execute-only database role for least-privilege callers. +7. Keep the SQL contract independent of any API, Geneva action, or other caller implementation. + +## Non-goals + +- Retrieving or capturing an actual execution plan. +- Reconstructing or executing SQL from Query Store. +- Returning statistics histograms, density vectors, or sampled column values. +- Clearing the procedure cache, updating statistics, forcing plans, or changing Query Store configuration. +- Providing caller concurrency control, circuit breaking, command timeouts, artifact retention, or download policy. +- Supporting on-premises SQL Server or self-hosted deployments in the baseline. +- Versioning the result contracts independently of the FHIR database schema. + +## Platform and disclosure boundary + +### Azure SQL Database + +The baseline targets Azure SQL Database and uses only Query Store catalog columns guaranteed across the supported Azure SQL deployment fleet at implementation time. Optional columns that may be rolling out regionally must not be referenced until they are universally available. + +### Query Store plans are estimated plans + +`sys.query_store_plan.query_plan` contains the compile-time Showplan, equivalent to `SET SHOWPLAN_XML ON`. Query Store combines this plan with aggregated runtime statistics; it does not retain an actual plan for every execution. + +The baseline reserves the future procedure name: + +```text +dbo.GetLastActualQueryPlanDiagnostics +``` + +This is documentation only. No stub procedure, shared output contract, permission grant, Query Store text execution, `LAST_QUERY_PLAN_STATS` enablement, or plan-cache lookup is included. + +### Accepted query and plan content + +Query text, statement text, scalar expressions, non-parameter constants, object names, missing-index recommendations, warnings, memory grants, and optimizer statistics usage are permitted diagnostic output. + +This intentionally accepts that ad hoc or non-parameterized query text and plan constants may contain literal values. The protected content is parameter-value metadata contained in Showplan `ParameterList` elements, including compiled and runtime parameter values. + +Statistics histogram values remain excluded because `range_high_key` contains actual indexed-column values. + +## Stored procedures + +All procedures: + +- use `WITH EXECUTE AS 'dbo'`; +- use `SET NOCOUNT ON`; +- use no dynamic SQL; +- create no explicit transaction; +- do not change session isolation level, `LOCK_TIMEOUT`, or `XACT_ABORT`; +- return exactly one result set; +- use repository-standard `THROW` errors for invalid calls and unavailable prerequisites; and +- write Start, End, and Error events through `dbo.LogEvent`. + +### 1. `dbo.GetQueryStoreSlowQueries` + +Returns one row per `query_id + plan_id` for regular executions in Query Store runtime intervals overlapping the requested time range. + +#### Inputs + +| Parameter | Behavior | +|---|---| +| `@StartTime datetimeoffset = NULL` | Defaults to one hour before the resolved `@EndTime`. | +| `@EndTime datetimeoffset = NULL` | Defaults to `SYSUTCDATETIME()`. | +| `@Top int = 20` | Must be between 1 and 100. | +| `@Offset int = 0` | Must be between 0 and 10,000. `@Offset = 10000` may still return up to 100 rows. | +| `@OrderBy varchar(32) = 'TotalDuration'` | Case-insensitive allowlist described below. | +| `@MinExecutions bigint = 1` | Must be a positive `bigint`. | +| `@QueryTextContains nvarchar(256) = NULL` | Optional literal substring filter. After trimming, it must contain 3-256 characters. | + +`@StartTime` and `@EndTime` accept explicit offsets and are normalized to UTC. The start must precede the end, and the requested range must not exceed 24 hours. + +`@QueryTextContains`: + +- is the only query-content filter; +- is matched under the database collation; +- may contain any caller-supplied text; +- is treated as a literal substring, not a caller-defined `LIKE` pattern; +- escapes `~`, `%`, `_`, and `[` and uses an explicit `ESCAPE N'~'` clause; +- rejects whitespace-only values; and +- is never written to `dbo.LogEvent`. + +The `@OrderBy` allowlist is: + +- `TotalDuration` +- `AverageDuration` +- `MaximumDuration` +- `TotalCpu` +- `AverageCpu` +- `LogicalReads` +- `Executions` +- `TotalWait` + +Unknown order values fail explicitly. Ordering uses a static `CASE` expression rather than dynamic SQL. Diagnostic metrics sort descending, NULL wait totals sort last, and `query_id ASC, plan_id ASC` are deterministic tie-breakers. + +There is no execution-type input in the baseline. Runtime and wait metrics include regular executions only. The implementation should contain a focused comment identifying where execution-type support could be added later. + +#### Time-window semantics + +Query Store runtime rows are interval aggregates. The procedure includes every interval that overlaps the half-open requested range `[StartTime, EndTime)`. Edge intervals may therefore include executions immediately outside the requested timestamps. The result does not repeat the resolved request window or interval boundaries. + +#### Runtime aggregation + +Query Store can expose multiple in-memory and persisted rows for the active interval. Runtime data must first collapse rows by: + +```text +plan_id + execution_type + runtime_stats_interval_id +``` + +It is then rolled up by `query_id + plan_id`. + +Weighted totals and averages use `decimal(38,4)` intermediates: + +```text +total duration = SUM(avg_duration * count_executions) +average duration = total duration / SUM(count_executions) +``` + +The same weighting applies to CPU, reads, writes, and row count. Totals are returned as `decimal(38,0)` and averages as `decimal(38,4)`. + +Minimum values use the minimum of interval minima. Maximum values use the maximum of interval maxima. Last values come from the row with the latest execution time, using runtime interval ID and runtime-statistics row ID descending as deterministic tie-breakers. + +Query-level compile count and last compile time are repeated on each plan row and must be clearly named as query-level metadata. + +Plans with fewer than `@MinExecutions` regular executions in the selected window are excluded. The diagnostic procedures' own Query Store entries are also excluded. All other object-bound and ad hoc Query Store entries are eligible. + +#### Wait statistics + +Wait statistics are aggregated for the same regular-execution population and overlapping intervals as runtime metrics. + +Each result row contains: + +- `TotalWaitMilliseconds` +- `AverageWaitMilliseconds` +- `WaitStatsStatus` +- `WaitStatsXml` + +`WaitStatsXml` contains one element per wait category, ordered by total wait descending, with: + +- category name; +- total wait milliseconds; +- average wait milliseconds; and +- maximum wait milliseconds. + +Zero-wait categories are omitted. When wait capture is available but a plan has no waits, the value is an empty typed root such as ``. When wait capture is disabled or unavailable, `WaitStatsXml` and scalar wait metrics are NULL and `WaitStatsStatus` explains the condition. Other runtime results still return. + +If `@OrderBy = 'TotalWait'` while wait capture is disabled or unavailable, rows still return. NULL wait totals sort last. + +#### Output + +The single result set includes: + +- `query_id` +- `plan_id` +- `query_hash` +- `query_plan_hash` +- full `query_sql_text` +- `object_id` +- `object_name`, without a separate schema-name column +- regular execution count +- total, average, minimum, maximum, and last duration in explicitly named microsecond columns +- total and average CPU in explicitly named microsecond columns +- total and average logical reads +- total and average physical reads +- total and average logical writes +- average and maximum row count +- first and last execution time in UTC +- query-level compile count and last compile time +- forced-plan state and available force-failure metadata +- plan type and other universally available diagnostic plan metadata +- total and average wait milliseconds +- `WaitStatsStatus` +- `WaitStatsXml` + +Physical reads and writes are output metrics but are not ordering options. Query context/handle metadata and execution type are omitted. + +Readable Query Store `READ_WRITE` and `READ_ONLY` states return available data. The actual state and read-only reason are logged. `OFF`, `ERROR`, or otherwise unreadable states fail explicitly. A readable store with no qualifying rows returns no rows. + +### 2. `dbo.GetQueryStorePlanDiagnostics` + +Accepts one required `@PlanId bigint` and returns one row containing Query Store metadata, full query text, and a parameter-redacted Showplan. + +An unknown or evicted plan ID fails explicitly with a stable "plan not found or no longer retained" error. The procedure does not fall back to another plan or constrain the plan by a time range. + +#### Output + +The result includes: + +- `plan_id` +- `query_id` +- `query_hash` +- `query_plan_hash` +- full `query_sql_text` +- engine and compatibility versions +- compile metadata +- trivial, parallel, forced-plan, force-failure, plan-type, dispatcher, and query-variant metadata when available through universally supported Azure SQL columns +- first and last execution metadata when available +- `SanitizationStatus` +- `SanitizationErrorCode` +- `SanitizedShowPlanXml` + +The entire multi-statement Showplan document is preserved. There is no separate allowlisted `PlanDiagnosticsXml`. + +#### Showplan sanitization + +The raw Query Store plan must never be returned. The procedure: + +1. copies `query_plan` into a local `xml` variable; +2. counts all elements whose local name is `ParameterList`, regardless of namespace; +3. removes every such element using a bounded XML DML loop; +4. verifies structurally that no `ParameterList` element and no `ParameterCompiledValue` or `ParameterRuntimeValue` attribute remains; +5. serializes the result and performs a case-insensitive textual check for those forbidden names; and +6. returns the XML only when every verification succeeds. + +Unknown Showplan namespaces are processed using the same namespace-agnostic removal and verification. All content other than `ParameterList` elements is preserved, including statement text, non-parameter constants, object/index names, missing-index recommendations, warnings, memory grants, optimizer statistics usage, and plan shape. + +There is no serialized plan-size cap. + +#### Partial availability + +If the plan row exists but `query_plan` is NULL, return the safe metadata with: + +- `SanitizedShowPlanXml = NULL`; +- `SanitizationStatus = 'PlanXmlUnavailable'`; and +- a stable non-sensitive error code. + +If the XML cannot be parsed or redaction verification fails, return safe metadata only with: + +- `SanitizedShowPlanXml = NULL`; +- `SanitizationStatus = 'InvalidXml'` or `'VerificationFailed'`; and +- a stable non-sensitive error code. + +Detailed parser messages must not be returned because they may echo plan content. These conditions also write an Error audit event. Raw or partially sanitized XML is never returned as a fallback. + +### 3. `dbo.GetStatisticsHealth` + +Returns one row per statistics object for user tables. + +#### Inputs + +| Parameter | Behavior | +|---|---| +| `@TableName sysname = NULL` | Optional exact table-name filter under the database collation. | +| `@Top int = 20` | Must be between 1 and 100. | +| `@Offset int = 0` | Must be between 0 and 10,000. | +| `@OrderBy varchar(32) = 'ModificationPercent'` | Case-insensitive allowlist described below. | + +There is no statistics-name filter and no minimum modification count/percentage filter. + +A supplied table name must be nonblank and resolve to exactly one user table. Unknown names fail explicitly. The baseline assumes FHIR operational tables do not span multiple schemas, so table-name input and output omit schema. + +Database-wide results exclude temporal history tables. An exact `@TableName` request may explicitly select a temporal history table. + +The `@OrderBy` allowlist is: + +- `ModificationCount` +- `ModificationPercent` +- `LastUpdated` +- `SamplingPercent` +- `Rows` + +Unknown values fail explicitly. Modification count, modification percentage, sampling percentage, and rows sort descending. `LastUpdated` places NULL values first and then sorts oldest first. Table name and statistics name ascending are deterministic tie-breakers. + +#### Sources + +- `sys.tables` +- `sys.stats` +- `sys.stats_columns` +- `sys.columns` +- `sys.indexes` +- `sys.dm_db_stats_properties` + +The procedure includes index, user-created, and auto-created statistics. Memory-optimized tables are included when compatible metadata is available. Microsoft-shipped and internal tables are excluded. + +#### Output + +The result includes: + +- table name +- statistics name +- statistics ID +- ordered typed XML containing each statistics-column ordinal and name +- auto-created and user-created flags +- incremental, persisted-sample, and no-recompute flags +- filtered-statistics flag and full `filter_definition` +- associated index ID, name, and type description +- disabled and hypothetical index flags +- last update time in UTC +- decimal `HoursSinceLastUpdate` +- row and unfiltered-row counts +- sampled rows +- sampling percentage +- histogram step count, but not histogram contents +- modification counter +- uncapped modification percentage calculated as `modification_counter / rows` +- `StatisticsStatus` + +Sampling and modification percentages are NULL when their denominator is zero or required properties are unavailable. Modification percentages may exceed 100 percent. + +If `sys.dm_db_stats_properties` returns no row, the statistics object remains in the result with property fields NULL and `StatisticsStatus = 'PropertiesUnavailable'`. + +Incremental statistics expose only the incremental flag. Partition-level properties are out of scope. + +The procedure must not call `DBCC SHOW_STATISTICS`, `sys.dm_db_stats_histogram`, or any source that returns histogram keys or density vectors. + +## Security model + +### Database role + +Create the database role: + +```sql +CREATE ROLE FhirDiagnosticsReader; +``` + +Grant the role `EXECUTE` individually on: + +- `dbo.GetQueryStoreSlowQueries` +- `dbo.GetQueryStorePlanDiagnostics` +- `dbo.GetStatisticsHealth` + +Future diagnostic procedures require an explicit reviewed grant. Do not grant schema-level execution on `dbo`. + +The role receives no: + +- `db_datareader`; +- direct `SELECT` on FHIR tables; +- direct Query Store catalog access; +- `VIEW DATABASE STATE`; +- arbitrary command execution; or +- direct `EXECUTE` permission on `dbo.LogEvent`. + +Existing database administrators can use the procedures immediately through their existing privileges. The role is available for future least-privilege callers, with membership controlled independently in each environment. + +The "Reader" name describes the externally observable diagnostic behavior. Internal audit writes do not alter Query Store, statistics, plan cache, FHIR data, or schema. + +### Caller integration + +The initial caller remains undecided. A direct SQL connection, PaaS administrative operation, Geneva action, or other approved tool may invoke the same SQL contract. + +Caller authentication, authorization, concurrency control, circuit breaking, command timeout, result retention, and download policy are caller responsibilities. Returned query text and sanitized Showplan are treated as operational metadata, but full artifacts must not be written to general logs, metrics dimensions, or `dbo.LogEvent`. + +## Resource controls + +SQL enforces: + +- a default one-hour and hard maximum 24-hour slow-query window; +- `@Top <= 100`; +- `@Offset <= 10000`; +- a positive `@MinExecutions`; +- a 3-256-character literal query-text substring; +- one plan per plan-diagnostics call; +- static SQL only; +- regular-execution-only runtime and wait aggregation; and +- exclusion of the diagnostic procedures' own Query Store entries. + +Limits are hard-coded in the procedures. There is no `dbo.Parameters` kill switch, SQL concurrency gate, plan-size cap, total-count query, continuation token, or `HasMoreRows` result. + +## Audit and observability + +Every procedure follows the existing `dbo.LogEvent` pattern and writes Start, End, and Error events, including successful calls. + +Audit records include: + +- `ORIGINAL_LOGIN()`; +- effective database principal from `USER_NAME()`; +- procedure name; +- bounded request metadata; +- elapsed milliseconds; +- returned row count; and +- sanitized XML size for successful plan retrieval. + +Slow-query audit metadata includes resolved UTC window, ordering mode, `@Top`, `@Offset`, `@MinExecutions`, and query-text filter presence/length, but never the filter text. + +Plan audit metadata includes `plan_id`, sanitization status, stable error code, and result size, but never query text or XML. + +Statistics audit metadata includes the exact validated table name when supplied, ordering mode, `@Top`, and `@Offset`. + +Audit records must not contain: + +- query text; +- `@QueryTextContains`; +- Showplan XML; +- parameter values; +- histogram values; or +- every query/plan ID returned by a page. + +If Start, End, or Error logging fails, the diagnostic call fails. Callers do not receive direct permission to invoke `dbo.LogEvent`. + +## Testing requirements + +### Slow-query aggregation + +1. Duplicate active-interval in-memory/persisted rows are collapsed before rollup. +2. Weighted totals and averages use the agreed decimal precision. +3. Minimum, maximum, and deterministic last-value calculations are correct. +4. Only regular executions contribute to runtime and wait metrics. +5. Overlapping Query Store interval semantics are verified at both window boundaries. +6. Time, row, offset, minimum-execution, query-text, and order allowlists cannot be bypassed. +7. Literal query-text matching correctly escapes `~`, `%`, `_`, and `[`. +8. Query Store `READ_WRITE` and readable `READ_ONLY` states return data. +9. Query Store `OFF`, `ERROR`, and unreadable states fail with actionable errors. +10. Wait capture available, disabled, unavailable, empty, and `TotalWait` ordering cases are covered. +11. Diagnostic procedures exclude their own Query Store entries. + +### Showplan sanitization + +1. Fixtures include single- and multi-statement plans. +2. Fixtures include compiled values, runtime values, multiple `ParameterList` elements, plans without parameters, unusual namespaces/extensions, PSP/variant plans when available, large/deep plans, and malformed XML. +3. Fixtures contain PHI-shaped parameter values. +4. Serialized output contains no `ParameterList`, `ParameterCompiledValue`, or `ParameterRuntimeValue`. +5. Statement text, non-parameter constants, missing-index recommendations, warnings, and other non-parameter content remain unchanged. +6. Unknown namespaces sanitize successfully when verification passes. +7. NULL, malformed, and verification-failing XML returns metadata only with the correct stable status/code. +8. Raw or partially sanitized XML is never returned. +9. Representative sanitized plans are manually verified to open in the SSMS graphical plan viewer. + +### Statistics + +1. All statistics types are returned with correct ordered column XML. +2. Filter definitions, index metadata, disabled/hypothetical flags, and missing-property status are correct. +3. Sampling/modification percentage zero-denominator behavior is correct. +4. Modification percentages above 100 percent are preserved. +5. Database-wide temporal-history exclusion and explicit history-table inclusion are covered. +6. Histogram keys and density vectors never appear. + +### Permissions and integration + +1. A principal with `FhirDiagnosticsReader` can execute all three procedures. +2. Procedures use `EXECUTE AS 'dbo'` and capture both original and effective identities. +3. Start, End, and Error audit behavior is verified, including fail-closed logging failures. +4. Procedures remain read-only except for required audit events. +5. Deterministic fixture tests are supplemented by Azure SQL integration tests for live catalog compatibility. + +## Schema and rollout + +- Add all three procedures under `src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs`. +- Add an idempotent role and individual permission migration. +- Introduce all three procedures and the role in one Azure SQL database schema version and migration diff. +- Deploy the schema objects consistently across supported environments; control role membership per environment. +- Use normal repository code review and automated/manual validation. No separate security-review gate is required. +- Do not add self-hosted/on-premises SQL Server support in this baseline. +- Keep actual-plan diagnostics as a documented future feature only. + +## References + +- [Monitor performance by using Query Store](https://learn.microsoft.com/sql/relational-databases/performance/monitoring-performance-by-using-the-query-store) +- [How Query Store collects data](https://learn.microsoft.com/sql/relational-databases/performance/how-query-store-collects-data) +- [`sys.query_store_runtime_stats`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-runtime-stats-transact-sql) +- [`sys.query_store_wait_stats`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-wait-stats-transact-sql) +- [`sys.query_store_plan`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-plan-transact-sql) +- [`sys.query_store_query`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-query-transact-sql) +- [`sys.query_store_query_text`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-query-text-transact-sql) +- [`sys.database_query_store_options`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-database-query-store-options-transact-sql) +- [`sys.dm_exec_query_plan_stats`](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-views/sys-dm-exec-query-plan-stats-transact-sql) +- [`sys.dm_db_stats_properties`](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-views/sys-dm-db-stats-properties-transact-sql) +- [`sys.dm_db_stats_histogram`](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-views/sys-dm-db-stats-histogram-transact-sql) +- [Showplan XML schemas](https://schemas.microsoft.com/sqlserver/2004/07/showplan/) From 629275c7fb1ba253ee89860156212eb7f92e4c8f Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 13 Aug 2026 19:27:40 +0000 Subject: [PATCH 02/20] Clarify diagnostics repository ownership Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 68 ++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 2efdb49377..7a7af450ea 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -2,7 +2,7 @@ ## Status -Agreed baseline for implementation. This document defines the SQL contract, security boundary, operational limits, and validation requirements. It does not select a Geneva action, PaaS API, or direct-SQL caller. +Agreed baseline for implementation. This document defines the SQL contract, security boundary, operational limits, validation requirements, and repository ownership split. It does not select a Geneva action, PaaS API, or direct-SQL caller. ## Problem @@ -15,6 +15,8 @@ FHIR Azure SQL performance investigations currently require privileged, manual a The baseline is a self-contained, read-only SQL interface. Filtering, validation, redaction, paging, permissions, and auditing must live in SQL so the procedures can be used by an authorized direct SQL connection or wrapped by future operational tooling. +The canonical database objects belong to the OSS `fhir-server` schema because that repository owns the versioned SQL migration chain consumed by FHIR PaaS. PaaS owns how authorized operators invoke the contract and handle its results. + Related internal guidance: - `Health.wiki/Home/Olympus-Team/DRI/TSGs/SQL-Latency-Issues.md` @@ -68,6 +70,49 @@ This intentionally accepts that ad hoc or non-parameterized query text and plan Statistics histogram values remain excluded because `range_high_key` contains actual indexed-column values. +## Repository ownership + +### OSS `fhir-server` + +The OSS repository owns the persistent database contract: + +- stored procedure definitions; +- `FhirDiagnosticsReader`; +- individual procedure grants; +- schema version and migration scripts; +- SQL aggregation, sanitization, permission, and compatibility tests; and +- the canonical SQL interface documentation. + +These objects must be part of the normal `Microsoft.Health.Fhir.SqlServer` schema artifacts and applied by `Microsoft.Health.Fhir.SchemaManager`. A database at the corresponding schema version must not depend on a separate PaaS rollout to acquire them. + +Although the supported operational scenario is FHIR PaaS on Azure SQL Database, the OSS migration must remain safe for databases that consume the OSS SQL schema. PaaS-specific identities, storage accounts, APIs, and rollout mechanisms must not be embedded in the OSS procedures. + +### `fhir-paas` + +The PaaS repository owns the operational integration: + +- selecting the initial caller surface, such as direct support tooling, Script Runner, Geneva, or a PaaS administrative operation; +- mapping an approved managed identity or support principal to `FhirDiagnosticsReader`; +- caller authentication and authorization; +- operation-level concurrency control, circuit breaking, and command timeout; +- invoking the OSS stored procedures without caller-supplied SQL; +- formatting, transporting, retaining, and auditing downloaded result artifacts; and +- coordinating deployment after the required OSS package/schema version is available. + +PaaS must consume the procedures through the OSS `Microsoft.Health.Fhir.SqlServer` and `Microsoft.Health.Fhir.SchemaManager` packages. It must not maintain a second PaaS-only schema version or duplicate production `CREATE OR ALTER PROCEDURE` definitions. + +The PaaS Script Runner may be used for a temporary read-only prototype or to invoke the deployed stored procedures. It must not be the production installation mechanism for these persistent objects. Otherwise a database could report the current OSS schema version while silently lacking the diagnostic procedures or role. + +### Rollout dependency + +The rollout order is: + +1. merge and release the OSS schema change; +2. update `fhir-paas` to consume the OSS package containing that schema version; +3. allow the existing PaaS schema manager flow to apply the migration; +4. provision approved role membership; and +5. enable the PaaS invocation and artifact-handling workflow. + ## Stored procedures All procedures: @@ -369,6 +414,8 @@ The initial caller remains undecided. A direct SQL connection, PaaS administrati Caller authentication, authorization, concurrency control, circuit breaking, command timeout, result retention, and download policy are caller responsibilities. Returned query text and sanitized Showplan are treated as operational metadata, but full artifacts must not be written to general logs, metrics dimensions, or `dbo.LogEvent`. +For the managed PaaS service, these caller responsibilities are implemented in `fhir-paas`; they are not added to the OSS schema migration. + ## Resource controls SQL enforces: @@ -460,17 +507,30 @@ If Start, End, or Error logging fails, the diagnostic call fails. Callers do not 3. Start, End, and Error audit behavior is verified, including fail-closed logging failures. 4. Procedures remain read-only except for required audit events. 5. Deterministic fixture tests are supplemented by Azure SQL integration tests for live catalog compatibility. +6. OSS tests verify the persistent SQL contract without depending on PaaS assemblies or infrastructure. +7. PaaS tests verify package/schema-version synchronization, role provisioning, stored-procedure invocation, and artifact handling without duplicating the SQL implementation. ## Schema and rollout +### OSS schema change + - Add all three procedures under `src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs`. - Add an idempotent role and individual permission migration. -- Introduce all three procedures and the role in one Azure SQL database schema version and migration diff. -- Deploy the schema objects consistently across supported environments; control role membership per environment. +- Introduce all three procedures and the role in one database schema version and migration diff. +- Include the objects in the generated full schema and packaged SchemaManager resources. +- Keep the migration additive and compatible with the previous application release. - Use normal repository code review and automated/manual validation. No separate security-review gate is required. -- Do not add self-hosted/on-premises SQL Server support in this baseline. - Keep actual-plan diagnostics as a documented future feature only. +### PaaS integration change + +- Update the OSS FHIR package versions and synchronized target schema version through the existing `fhir-paas` dependency flow. +- Do not copy the stored procedure or role DDL into PaaS Script Runner scripts. +- Add role membership only for the approved operational identity. +- Implement the selected caller, result transport, artifact storage, and operational authorization in `fhir-paas`. +- Deploy schema/package consumption before enabling the caller. +- Control caller rollout and role membership independently in each environment. + ## References - [Monitor performance by using Query Store](https://learn.microsoft.com/sql/relational-databases/performance/monitoring-performance-by-using-the-query-store) From 24bc3577ab4120decad3aa9351075203fe4b3598 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 13 Aug 2026 21:36:02 +0000 Subject: [PATCH 03/20] Implement Query Store performance diagnostics Add schema version 117 with bounded Query Store and statistics procedures, least-privilege grants, and SQL-backed integration coverage.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 24 +- .../Features/Schema/Migrations/117.diff.sql | 871 ++++++++++++++++++ .../Features/Schema/SchemaVersion.cs | 1 + .../Features/Schema/SchemaVersionConstants.cs | 2 +- .../Schema/Sql/Scripts/DiagnosticsRole.sql | 26 + .../Sprocs/GetQueryStorePlanDiagnostics.sql | 229 +++++ .../Sql/Sprocs/GetQueryStoreSlowQueries.sql | 392 ++++++++ .../Schema/Sql/Sprocs/GetStatisticsHealth.sql | 224 +++++ .../Microsoft.Health.Fhir.SqlServer.csproj | 3 +- ...th.Fhir.Shared.Tests.Integration.projitems | 1 + .../SqlServerQueryStoreDiagnosticsTests.cs | 239 +++++ 11 files changed, 2002 insertions(+), 10 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/DiagnosticsRole.sql create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql create mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 7a7af450ea..99d3eca0f8 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -62,6 +62,11 @@ dbo.GetLastActualQueryPlanDiagnostics This is documentation only. No stub procedure, shared output contract, permission grant, Query Store text execution, `LAST_QUERY_PLAN_STATS` enablement, or plan-cache lookup is included. +## Implementation simplifications + +- Plan-type and Parameter Sensitive Plan dispatcher/query-variant metadata are intentionally not read. Those Azure SQL catalog fields are not stable across the supported deployment fleet, so both Query Store procedures omit them rather than returning speculative NULL/status fields or attempting version-specific fallback logic. +- `GetStatisticsHealth` reports database-level `sys.dm_db_stats_properties` metadata for each statistics object. It does not expand incremental statistics into partition-level property rows; unavailable properties remain `NULL` and are explicitly marked `PropertiesUnavailable`. Its table-name input is materialized as `nvarchar(128)`, rather than the type-equivalent `sysname` alias, because the existing schema C# model generator interprets `sysname` as a table-valued parameter. + ### Accepted query and plan content Query text, statement text, scalar expressions, non-parameter constants, object names, missing-index recommendations, warnings, memory grants, and optimizer statistics usage are permitted diagnostic output. @@ -241,7 +246,7 @@ The single result set includes: - first and last execution time in UTC - query-level compile count and last compile time - forced-plan state and available force-failure metadata -- plan type and other universally available diagnostic plan metadata +- other universally available diagnostic plan metadata; plan-type, dispatcher, and query-variant metadata are omitted - total and average wait milliseconds - `WaitStatsStatus` - `WaitStatsXml` @@ -260,20 +265,21 @@ An unknown or evicted plan ID fails explicitly with a stable "plan not found or The result includes: -- `plan_id` -- `query_id` -- `query_hash` -- `query_plan_hash` -- full `query_sql_text` +- `PlanId` +- `QueryId` +- `QueryHash` +- `QueryPlanHash` +- full `QuerySqlText` - engine and compatibility versions - compile metadata -- trivial, parallel, forced-plan, force-failure, plan-type, dispatcher, and query-variant metadata when available through universally supported Azure SQL columns +- trivial, parallel, forced-plan, and force-failure metadata - first and last execution metadata when available - `SanitizationStatus` - `SanitizationErrorCode` - `SanitizedShowPlanXml` The entire multi-statement Showplan document is preserved. There is no separate allowlisted `PlanDiagnosticsXml`. +Plan-type, dispatcher, and query-variant metadata are unavailable on the baseline catalog and are omitted. #### Showplan sanitization @@ -314,7 +320,7 @@ Returns one row per statistics object for user tables. | Parameter | Behavior | |---|---| -| `@TableName sysname = NULL` | Optional exact table-name filter under the database collation. | +| `@TableName nvarchar(128) = NULL` | Optional exact table-name filter under the database collation. | | `@Top int = 20` | Must be between 1 and 100. | | `@Offset int = 0` | Must be between 0 and 10,000. | | `@OrderBy varchar(32) = 'ModificationPercent'` | Case-insensitive allowlist described below. | @@ -465,6 +471,8 @@ If Start, End, or Error logging fails, the diagnostic call fails. Callers do not ## Testing requirements +**Deferred prototype validation:** The prototype has one representative SQL-backed end-to-end path. Exhaustive matrix validation remains future work, including negative validation, fail-closed audit behavior, permissions, a malformed-fixture corpus, and wait-disabled cases. + ### Slow-query aggregation 1. Duplicate active-interval in-memory/persisted rows are collapsed before rollup. diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql new file mode 100644 index 0000000000..a421077cc2 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql @@ -0,0 +1,871 @@ +IF NOT EXISTS +( + SELECT 1 + FROM sys.database_principals + WHERE name = N'FhirDiagnosticsReader' + AND type = 'R' +) +BEGIN + IF EXISTS + ( + SELECT 1 + FROM sys.database_principals + WHERE name = N'FhirDiagnosticsReader' + ) + BEGIN + THROW 50100, 'A database principal named FhirDiagnosticsReader already exists but is not a database role.', 1; + END + + CREATE ROLE [FhirDiagnosticsReader]; +END +GO + +CREATE OR ALTER PROCEDURE dbo.GetQueryStoreSlowQueries + @StartTime datetimeoffset(7) = NULL + ,@EndTime datetimeoffset(7) = NULL + ,@Top int = 20 + ,@Offset int = 0 + ,@OrderBy varchar(32) = 'TotalDuration' + ,@MinExecutions bigint = 1 + ,@QueryTextContains nvarchar(256) = NULL +WITH EXECUTE AS 'dbo' +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @ProcedureName varchar(100) = OBJECT_NAME(@@PROCID); + DECLARE @AuditMode varchar(200) = 'QueryStoreSlowQueries'; + DECLARE @AuditStartTime datetime = GETUTCDATE(); + DECLARE @AuditText nvarchar(3500); + DECLARE @RowsReturned bigint; + DECLARE @ResolvedStartTime datetimeoffset(7); + DECLARE @ResolvedEndTime datetimeoffset(7); + DECLARE @OrderByNormalized varchar(32); + DECLARE @QueryTextPattern nvarchar(514); + DECLARE @QueryTextFilterLength int; + DECLARE @QueryStoreState nvarchar(60); + DECLARE @QueryStoreReadOnlyReason bigint; + DECLARE @WaitStatsCaptureMode nvarchar(60); + DECLARE @WaitStatsStatus varchar(32); + DECLARE @SlowQueriesProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStoreSlowQueries'); + DECLARE @PlanDiagnosticsProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStorePlanDiagnostics'); + DECLARE @StatisticsHealthProcedureObjectId int = OBJECT_ID(N'dbo.GetStatisticsHealth'); + + IF @ProcedureName IS NULL + SET @ProcedureName = 'GetQueryStoreSlowQueries'; + + SET @AuditText = CONCAT( + N'OriginalLogin=', ORIGINAL_LOGIN(), + N';EffectivePrincipal=', USER_NAME()); + + BEGIN TRY + SET @Top = ISNULL(@Top, 20); + SET @Offset = ISNULL(@Offset, 0); + SET @MinExecutions = ISNULL(@MinExecutions, 1); + SET @OrderBy = ISNULL(@OrderBy, 'TotalDuration'); + SET @ResolvedEndTime = SWITCHOFFSET(ISNULL(@EndTime, TODATETIMEOFFSET(SYSUTCDATETIME(), '+00:00')), '+00:00'); + SET @ResolvedStartTime = SWITCHOFFSET(ISNULL(@StartTime, DATEADD(hour, -1, @ResolvedEndTime)), '+00:00'); + SET @QueryTextContains = NULLIF(LTRIM(RTRIM(@QueryTextContains)), N''); + SET @QueryTextFilterLength = ISNULL(LEN(@QueryTextContains), 0); + + SELECT + @QueryStoreState = actual_state_desc, + @QueryStoreReadOnlyReason = readonly_reason, + @WaitStatsCaptureMode = wait_stats_capture_mode_desc + FROM sys.database_query_store_options; + + SET @WaitStatsStatus = + CASE @WaitStatsCaptureMode + WHEN N'ON' THEN 'Available' + WHEN N'OFF' THEN 'Disabled' + ELSE 'Unavailable' + END; + SET @AuditText = CONCAT( + @AuditText, + N';StartTimeUtc=', CONVERT(nvarchar(33), @ResolvedStartTime, 127), + N';EndTimeUtc=', CONVERT(nvarchar(33), @ResolvedEndTime, 127), + N';OrderBy=', @OrderBy, + N';Top=', @Top, + N';Offset=', @Offset, + N';MinExecutions=', @MinExecutions, + N';QueryTextFilterPresent=', CASE WHEN @QueryTextContains IS NULL THEN N'0' ELSE N'1' END, + N';QueryTextFilterLength=', @QueryTextFilterLength, + N';QueryStoreState=', ISNULL(@QueryStoreState, N'Unknown'), + N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown'), + N';WaitStatsCaptureMode=', ISNULL(@WaitStatsCaptureMode, N'Unknown')); + + EXECUTE dbo.LogEvent + @Process = @ProcedureName, + @Mode = @AuditMode, + @Status = 'Start', + @Text = @AuditText; + + IF @Top < 1 OR @Top > 100 + THROW 50400, '@Top must be between 1 and 100.', 1; + + IF @Offset < 0 OR @Offset > 10000 + THROW 50401, '@Offset must be between 0 and 10000.', 1; + + IF @MinExecutions < 1 + THROW 50402, '@MinExecutions must be positive.', 1; + + IF @ResolvedStartTime >= @ResolvedEndTime + THROW 50403, '@StartTime must precede @EndTime.', 1; + + IF @ResolvedEndTime > DATEADD(hour, 24, @ResolvedStartTime) + THROW 50404, 'The requested time range must not exceed 24 hours.', 1; + + IF @QueryTextContains IS NOT NULL + AND LEN(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(@QueryTextContains, N' ', N''), NCHAR(9), N''), NCHAR(10), N''), NCHAR(13), N''), NCHAR(160), N'')) = 0 + THROW 50405, '@QueryTextContains must not be whitespace only.', 1; + + IF @QueryTextContains IS NOT NULL + AND (@QueryTextFilterLength < 3 OR @QueryTextFilterLength > 256) + THROW 50406, '@QueryTextContains must contain between 3 and 256 characters after trimming.', 1; + + SET @OrderByNormalized = + CASE + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalDuration' THEN 'TotalDuration' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageDuration' THEN 'AverageDuration' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'MaximumDuration' THEN 'MaximumDuration' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalCpu' THEN 'TotalCpu' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageCpu' THEN 'AverageCpu' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'LogicalReads' THEN 'LogicalReads' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'Executions' THEN 'Executions' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalWait' THEN 'TotalWait' + END; + + IF @OrderByNormalized IS NULL + THROW 50407, '@OrderBy is not supported.', 1; + + IF ISNULL(@QueryStoreState, N'') NOT IN (N'READ_WRITE', N'READ_ONLY') + THROW 50408, 'Query Store is not enabled and readable.', 1; + + SET @QueryTextPattern = + CASE + WHEN @QueryTextContains IS NULL THEN NULL + ELSE N'%' + REPLACE(REPLACE(REPLACE(REPLACE(@QueryTextContains, N'~', N'~~'), N'%', N'~%'), N'_', N'~_'), N'[', N'~[') + N'%' + END; + + ;WITH RuntimeStatsRows AS + ( + SELECT + rs.plan_id AS PlanId, + rs.execution_type AS ExecutionType, + rs.runtime_stats_interval_id AS RuntimeStatsIntervalId, + rs.runtime_stats_id AS RuntimeStatsId, + rs.count_executions AS RegularExecutionCount, + CONVERT(decimal(38, 4), rs.avg_duration) AS AverageDurationMicroseconds, + CONVERT(decimal(38, 0), rs.min_duration) AS MinimumDurationMicroseconds, + CONVERT(decimal(38, 0), rs.max_duration) AS MaximumDurationMicroseconds, + CONVERT(decimal(38, 0), rs.last_duration) AS LastDurationMicroseconds, + CONVERT(decimal(38, 4), rs.avg_cpu_time) AS AverageCpuMicroseconds, + CONVERT(decimal(38, 4), rs.avg_logical_io_reads) AS AverageLogicalReads, + CONVERT(decimal(38, 4), rs.avg_physical_io_reads) AS AveragePhysicalReads, + CONVERT(decimal(38, 4), rs.avg_logical_io_writes) AS AverageLogicalWrites, + CONVERT(decimal(38, 4), rs.avg_rowcount) AS AverageRowCount, + CONVERT(decimal(38, 0), rs.max_rowcount) AS MaximumRowCount, + SWITCHOFFSET(rs.first_execution_time, '+00:00') AS FirstExecutionTimeUtc, + SWITCHOFFSET(rs.last_execution_time, '+00:00') AS LastExecutionTimeUtc + FROM sys.query_store_runtime_stats AS rs + INNER JOIN sys.query_store_runtime_stats_interval AS rsi + ON rsi.runtime_stats_interval_id = rs.runtime_stats_interval_id + -- The baseline is regular executions only. An explicit execution-type input belongs here if added later. + WHERE rs.execution_type = 0 + AND rsi.start_time < @ResolvedEndTime + AND rsi.end_time > @ResolvedStartTime + ), + RankedRuntimeStatsRows AS + ( + SELECT + rs.*, + ROW_NUMBER() OVER + ( + PARTITION BY rs.PlanId, rs.ExecutionType, rs.RuntimeStatsIntervalId + ORDER BY rs.LastExecutionTimeUtc DESC, rs.RuntimeStatsIntervalId DESC, rs.RuntimeStatsId DESC + ) AS LastValueRank + FROM RuntimeStatsRows AS rs + ), + CollapsedRuntimeStats AS + ( + SELECT + rs.PlanId, + rs.ExecutionType, + rs.RuntimeStatsIntervalId, + SUM(rs.RegularExecutionCount) AS RegularExecutionCount, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageDurationMicroseconds * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalDurationMicroseconds, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageCpuMicroseconds * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalCpuMicroseconds, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageLogicalReads * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalLogicalReads, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AveragePhysicalReads * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalPhysicalReads, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageLogicalWrites * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalLogicalWrites, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageRowCount * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalRowCount, + MIN(rs.MinimumDurationMicroseconds) AS MinimumDurationMicroseconds, + MAX(rs.MaximumDurationMicroseconds) AS MaximumDurationMicroseconds, + MAX(rs.MaximumRowCount) AS MaximumRowCount, + MIN(rs.FirstExecutionTimeUtc) AS FirstExecutionTimeUtc, + MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastExecutionTimeUtc END) AS LastExecutionTimeUtc, + MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastDurationMicroseconds END) AS LastDurationMicroseconds, + MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.RuntimeStatsId END) AS LastRuntimeStatsId + FROM RankedRuntimeStatsRows AS rs + GROUP BY + rs.PlanId, + rs.ExecutionType, + rs.RuntimeStatsIntervalId + ), + RankedCollapsedRuntimeStats AS + ( + SELECT + rs.*, + ROW_NUMBER() OVER + ( + PARTITION BY rs.PlanId + ORDER BY rs.LastExecutionTimeUtc DESC, rs.RuntimeStatsIntervalId DESC, rs.LastRuntimeStatsId DESC + ) AS LastValueRank + FROM CollapsedRuntimeStats AS rs + ), + AggregatedRuntimeStats AS + ( + SELECT + rs.PlanId, + SUM(rs.RegularExecutionCount) AS RegularExecutionCount, + CONVERT(decimal(38, 0), SUM(rs.TotalDurationMicroseconds)) AS TotalDurationMicroseconds, + CONVERT(decimal(38, 4), SUM(rs.TotalDurationMicroseconds) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageDurationMicroseconds, + MIN(rs.MinimumDurationMicroseconds) AS MinimumDurationMicroseconds, + MAX(rs.MaximumDurationMicroseconds) AS MaximumDurationMicroseconds, + MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastDurationMicroseconds END) AS LastDurationMicroseconds, + CONVERT(decimal(38, 0), SUM(rs.TotalCpuMicroseconds)) AS TotalCpuMicroseconds, + CONVERT(decimal(38, 4), SUM(rs.TotalCpuMicroseconds) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageCpuMicroseconds, + CONVERT(decimal(38, 0), SUM(rs.TotalLogicalReads)) AS TotalLogicalReads, + CONVERT(decimal(38, 4), SUM(rs.TotalLogicalReads) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageLogicalReads, + CONVERT(decimal(38, 0), SUM(rs.TotalPhysicalReads)) AS TotalPhysicalReads, + CONVERT(decimal(38, 4), SUM(rs.TotalPhysicalReads) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AveragePhysicalReads, + CONVERT(decimal(38, 0), SUM(rs.TotalLogicalWrites)) AS TotalLogicalWrites, + CONVERT(decimal(38, 4), SUM(rs.TotalLogicalWrites) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageLogicalWrites, + CONVERT(decimal(38, 4), SUM(rs.TotalRowCount) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageRowCount, + MAX(rs.MaximumRowCount) AS MaximumRowCount, + MIN(rs.FirstExecutionTimeUtc) AS FirstExecutionTimeUtc, + MAX(rs.LastExecutionTimeUtc) AS LastExecutionTimeUtc + FROM RankedCollapsedRuntimeStats AS rs + GROUP BY rs.PlanId + HAVING SUM(rs.RegularExecutionCount) >= @MinExecutions + ), + WaitStatsRows AS + ( + SELECT + ws.plan_id AS PlanId, + ws.wait_category_desc AS WaitCategoryDescription, + CONVERT(decimal(38, 0), ws.total_query_wait_time_ms) AS TotalWaitMilliseconds, + CONVERT(decimal(38, 0), ws.max_query_wait_time_ms) AS MaximumWaitMilliseconds + FROM sys.query_store_wait_stats AS ws + INNER JOIN sys.query_store_runtime_stats_interval AS rsi + ON rsi.runtime_stats_interval_id = ws.runtime_stats_interval_id + WHERE @WaitStatsStatus = 'Available' + AND ws.execution_type = 0 + AND rsi.start_time < @ResolvedEndTime + AND rsi.end_time > @ResolvedStartTime + ), + WaitCategories AS + ( + SELECT + ws.PlanId, + ws.WaitCategoryDescription, + CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds, + MAX(ws.MaximumWaitMilliseconds) AS MaximumWaitMilliseconds + FROM WaitStatsRows AS ws + GROUP BY + ws.PlanId, + ws.WaitCategoryDescription + ), + AggregatedWaitStats AS + ( + SELECT + ws.PlanId, + CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds + FROM WaitCategories AS ws + GROUP BY ws.PlanId + ), + WaitStatsXml AS + ( + SELECT + wp.PlanId, + ( + SELECT + wc.WaitCategoryDescription AS [@Category], + wc.TotalWaitMilliseconds AS [@TotalWaitMilliseconds], + CONVERT(decimal(38, 4), wc.TotalWaitMilliseconds / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) AS [@AverageWaitMilliseconds], + wc.MaximumWaitMilliseconds AS [@MaximumWaitMilliseconds] + FROM WaitCategories AS wc + WHERE wc.PlanId = wp.PlanId + AND wc.TotalWaitMilliseconds > 0 + ORDER BY + wc.TotalWaitMilliseconds DESC, + wc.WaitCategoryDescription ASC + FOR XML PATH(N'WaitCategory'), ROOT(N'WaitStats'), TYPE + ) AS WaitStatsXml + FROM (SELECT DISTINCT PlanId FROM WaitCategories) AS wp + INNER JOIN AggregatedRuntimeStats AS ars + ON ars.PlanId = wp.PlanId + ) + SELECT + q.query_id AS QueryId, + p.plan_id AS PlanId, + q.query_hash AS QueryHash, + p.query_plan_hash AS QueryPlanHash, + qt.query_sql_text AS QuerySqlText, + q.object_id AS ObjectId, + OBJECT_NAME(q.object_id) AS ObjectName, + ars.RegularExecutionCount, + ars.TotalDurationMicroseconds, + ars.AverageDurationMicroseconds, + ars.MinimumDurationMicroseconds, + ars.MaximumDurationMicroseconds, + ars.LastDurationMicroseconds, + ars.TotalCpuMicroseconds, + ars.AverageCpuMicroseconds, + ars.TotalLogicalReads, + ars.AverageLogicalReads, + ars.TotalPhysicalReads, + ars.AveragePhysicalReads, + ars.TotalLogicalWrites, + ars.AverageLogicalWrites, + ars.AverageRowCount, + ars.MaximumRowCount, + ars.FirstExecutionTimeUtc, + ars.LastExecutionTimeUtc, + q.count_compiles AS QueryLevelCompileCount, + SWITCHOFFSET(q.last_compile_start_time, '+00:00') AS QueryLevelLastCompileTimeUtc, + p.is_forced_plan AS IsForcedPlan, + p.force_failure_count AS ForceFailureCount, + p.last_force_failure_reason AS LastForceFailureReason, + p.last_force_failure_reason_desc AS LastForceFailureReasonDescription, + p.plan_group_id AS PlanGroupId, + p.engine_version AS EngineVersion, + p.compatibility_level AS CompatibilityLevel, + p.is_online_index_plan AS IsOnlineIndexPlan, + p.is_trivial_plan AS IsTrivialPlan, + p.is_parallel_plan AS IsParallelPlan, + CASE + WHEN @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) + END AS TotalWaitMilliseconds, + CASE + WHEN @WaitStatsStatus = 'Available' THEN CONVERT(decimal(38, 4), ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) + END AS AverageWaitMilliseconds, + @WaitStatsStatus AS WaitStatsStatus, + CASE + WHEN @WaitStatsStatus = 'Available' THEN ISNULL(wsx.WaitStatsXml, CONVERT(xml, N'')) + END AS WaitStatsXml + FROM AggregatedRuntimeStats AS ars + INNER JOIN sys.query_store_plan AS p + ON p.plan_id = ars.PlanId + INNER JOIN sys.query_store_query AS q + ON q.query_id = p.query_id + INNER JOIN sys.query_store_query_text AS qt + ON qt.query_text_id = q.query_text_id + LEFT JOIN AggregatedWaitStats AS aws + ON aws.PlanId = ars.PlanId + LEFT JOIN WaitStatsXml AS wsx + ON wsx.PlanId = ars.PlanId + WHERE ISNULL(q.object_id, -1) <> ISNULL(@SlowQueriesProcedureObjectId, -2) + AND ISNULL(q.object_id, -1) <> ISNULL(@PlanDiagnosticsProcedureObjectId, -2) + AND ISNULL(q.object_id, -1) <> ISNULL(@StatisticsHealthProcedureObjectId, -2) + AND (@QueryTextContains IS NULL OR qt.query_sql_text LIKE @QueryTextPattern ESCAPE N'~') + ORDER BY + CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' AND aws.TotalWaitMilliseconds IS NULL THEN 1 ELSE 0 END ASC, + CASE WHEN @OrderByNormalized = 'TotalDuration' THEN ars.TotalDurationMicroseconds END DESC, + CASE WHEN @OrderByNormalized = 'AverageDuration' THEN ars.AverageDurationMicroseconds END DESC, + CASE WHEN @OrderByNormalized = 'MaximumDuration' THEN ars.MaximumDurationMicroseconds END DESC, + CASE WHEN @OrderByNormalized = 'TotalCpu' THEN ars.TotalCpuMicroseconds END DESC, + CASE WHEN @OrderByNormalized = 'AverageCpu' THEN ars.AverageCpuMicroseconds END DESC, + CASE WHEN @OrderByNormalized = 'LogicalReads' THEN ars.TotalLogicalReads END DESC, + CASE WHEN @OrderByNormalized = 'Executions' THEN ars.RegularExecutionCount END DESC, + CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) END DESC, + q.query_id ASC, + p.plan_id ASC + OFFSET @Offset ROWS FETCH NEXT @Top ROWS ONLY; + + SET @RowsReturned = @@ROWCOUNT; + + EXECUTE dbo.LogEvent + @Process = @ProcedureName, + @Mode = @AuditMode, + @Status = 'End', + @Rows = @RowsReturned, + @Start = @AuditStartTime, + @Text = @AuditText; + END TRY + BEGIN CATCH + SET @AuditText = CONCAT( + @AuditText, + N';ErrorNumber=', ERROR_NUMBER(), + N';ErrorState=', ERROR_STATE()); + + EXECUTE dbo.LogEvent + @Process = @ProcedureName, + @Mode = @AuditMode, + @Status = 'Error', + @Start = @AuditStartTime, + @Text = @AuditText; + + THROW; + END CATCH +END +GO + +CREATE OR ALTER PROCEDURE dbo.GetQueryStorePlanDiagnostics @PlanId bigint +WITH EXECUTE AS 'dbo' +AS +BEGIN + SET NOCOUNT ON + + DECLARE @SP varchar(100) = OBJECT_NAME(@@PROCID) + ,@Mode varchar(200) = 'QueryStorePlanDiagnostics' + ,@Start datetime = GETUTCDATE() + ,@Rows int = 0 + ,@QueryStoreState nvarchar(60) + ,@QueryStoreReadonlyReason bigint + ,@AuditText nvarchar(3500) + ,@FoundPlanId bigint + ,@QueryId bigint + ,@QueryHash binary(8) + ,@QueryPlanHash binary(8) + ,@QuerySqlText nvarchar(max) + ,@ObjectId int + ,@PlanGroupId bigint + ,@EngineVersion nvarchar(128) + ,@CompatibilityLevel smallint + ,@IsOnlineIndexPlan bit + ,@IsTrivialPlan bit + ,@IsParallelPlan bit + ,@IsForcedPlan bit + ,@ForceFailureCount bigint + ,@LastForceFailureReason int + ,@LastForceFailureReasonDesc nvarchar(256) + ,@CountCompiles bigint + ,@InitialCompileStartTime datetimeoffset(7) + ,@LastCompileStartTime datetimeoffset(7) + ,@LastPlanExecutionTime datetimeoffset(7) + ,@AverageCompileDuration float + ,@LastCompileDuration bigint + ,@FirstExecutionTime datetimeoffset(7) + ,@LastRuntimeExecutionTime datetimeoffset(7) + ,@RawQueryPlan nvarchar(max) + ,@LocalPlanXml xml + ,@SanitizedShowPlanXml xml + ,@SerializedPlanXml nvarchar(max) + ,@ParameterListCount bigint + ,@ParameterListRemoved bigint = 0 + ,@RemainingParameterListCount bigint + ,@ForbiddenAttributeCount bigint + ,@SanitizationStatus varchar(32) + ,@SanitizationErrorCode varchar(64) + ,@SerializedResultSizeBytes bigint = 0 + ,@CaughtErrorNumber int + ,@CaughtErrorState int + + SET @AuditText = CONCAT( + N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), + N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), + N';PlanId=', ISNULL(CONVERT(nvarchar(20), @PlanId), N'NULL')) + + BEGIN TRY + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Start',@Text=@AuditText + + IF @PlanId IS NULL OR @PlanId <= 0 + THROW 50001, 'Plan ID must be a positive bigint.', 1 + + SELECT @QueryStoreState = actual_state_desc + ,@QueryStoreReadonlyReason = readonly_reason + FROM sys.database_query_store_options + + SET @AuditText = CONCAT( + @AuditText, + N';QueryStoreState=', ISNULL(CONVERT(nvarchar(60), @QueryStoreState), N'Unknown'), + N';QueryStoreReadonlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadonlyReason), N'NULL')) + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Run',@Text=@AuditText + + IF @QueryStoreState IS NULL OR @QueryStoreState NOT IN ('READ_WRITE', 'READ_ONLY') + THROW 50002, 'Query Store is not readable.', 1 + + SELECT @FoundPlanId = p.plan_id + ,@QueryId = p.query_id + ,@QueryHash = q.query_hash + ,@QueryPlanHash = p.query_plan_hash + ,@QuerySqlText = qt.query_sql_text + ,@ObjectId = q.object_id + ,@PlanGroupId = p.plan_group_id + ,@EngineVersion = p.engine_version + ,@CompatibilityLevel = p.compatibility_level + ,@IsOnlineIndexPlan = p.is_online_index_plan + ,@IsTrivialPlan = p.is_trivial_plan + ,@IsParallelPlan = p.is_parallel_plan + ,@IsForcedPlan = p.is_forced_plan + ,@ForceFailureCount = p.force_failure_count + ,@LastForceFailureReason = p.last_force_failure_reason + ,@LastForceFailureReasonDesc = p.last_force_failure_reason_desc + ,@CountCompiles = p.count_compiles + ,@InitialCompileStartTime = p.initial_compile_start_time + ,@LastCompileStartTime = p.last_compile_start_time + ,@LastPlanExecutionTime = p.last_execution_time + ,@AverageCompileDuration = p.avg_compile_duration + ,@LastCompileDuration = p.last_compile_duration + ,@RawQueryPlan = CONVERT(nvarchar(max), p.query_plan) + FROM sys.query_store_plan AS p + LEFT JOIN sys.query_store_query AS q + ON q.query_id = p.query_id + LEFT JOIN sys.query_store_query_text AS qt + ON qt.query_text_id = q.query_text_id + WHERE p.plan_id = @PlanId + + IF @FoundPlanId IS NULL + THROW 50003, 'The requested Query Store plan was not found or is no longer retained.', 1 + + SELECT @FirstExecutionTime = MIN(first_execution_time) + ,@LastRuntimeExecutionTime = MAX(last_execution_time) + FROM sys.query_store_runtime_stats + WHERE plan_id = @PlanId + + IF @RawQueryPlan IS NULL + BEGIN + SET @SanitizationStatus = 'PlanXmlUnavailable' + SET @SanitizationErrorCode = 'PLAN_XML_UNAVAILABLE' + END + ELSE + BEGIN + SET @LocalPlanXml = TRY_CONVERT(xml, @RawQueryPlan) + + IF @LocalPlanXml IS NULL + BEGIN + SET @SanitizationStatus = 'InvalidXml' + SET @SanitizationErrorCode = 'PLAN_XML_INVALID' + END + ELSE + BEGIN + BEGIN TRY + SET @SanitizedShowPlanXml = @LocalPlanXml + SET @ParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') + + WHILE @ParameterListRemoved < @ParameterListCount + BEGIN + SET @SanitizedShowPlanXml.modify('delete (//*[local-name(.) = "ParameterList"])[1]') + SET @ParameterListRemoved = @ParameterListRemoved + 1 + END + + SET @RemainingParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') + SET @ForbiddenAttributeCount = @SanitizedShowPlanXml.value('count(//@*[local-name(.) = "ParameterCompiledValue" or local-name(.) = "ParameterRuntimeValue"])', 'bigint') + SET @SerializedPlanXml = CONVERT(nvarchar(max), @SanitizedShowPlanXml) + + IF @RemainingParameterListCount = 0 + AND @ForbiddenAttributeCount = 0 + AND CHARINDEX(N'PARAMETERLIST', UPPER(@SerializedPlanXml)) = 0 + AND CHARINDEX(N'PARAMETERCOMPILEDVALUE', UPPER(@SerializedPlanXml)) = 0 + AND CHARINDEX(N'PARAMETERRUNTIMEVALUE', UPPER(@SerializedPlanXml)) = 0 + BEGIN + SET @SanitizationStatus = 'Sanitized' + END + ELSE + BEGIN + SET @SanitizedShowPlanXml = NULL + SET @SerializedPlanXml = NULL + SET @SanitizationStatus = 'VerificationFailed' + SET @SanitizationErrorCode = 'PLAN_XML_VERIFICATION_FAILED' + END + END TRY + BEGIN CATCH + SET @SanitizedShowPlanXml = NULL + SET @SerializedPlanXml = NULL + SET @SanitizationStatus = 'VerificationFailed' + SET @SanitizationErrorCode = 'PLAN_XML_VERIFICATION_FAILED' + END CATCH + END + END + + SET @SerializedResultSizeBytes = ISNULL(DATALENGTH(@SerializedPlanXml), 0) + SET @AuditText = CONCAT( + @AuditText, + N';SanitizationStatus=', ISNULL(CONVERT(nvarchar(32), @SanitizationStatus), N'NotStarted'), + N';SanitizationErrorCode=', ISNULL(CONVERT(nvarchar(64), @SanitizationErrorCode), N'NONE'), + N';SerializedResultSizeBytes=', CONVERT(nvarchar(20), @SerializedResultSizeBytes)) + + IF @SanitizationErrorCode IS NOT NULL + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Text=@AuditText + + SET @Rows = 1 + + SELECT @FoundPlanId AS PlanId + ,@QueryId AS QueryId + ,@QueryHash AS QueryHash + ,@QueryPlanHash AS QueryPlanHash + ,@QuerySqlText AS QuerySqlText + ,@ObjectId AS ObjectId + ,@PlanGroupId AS PlanGroupId + ,@EngineVersion AS EngineVersion + ,@CompatibilityLevel AS CompatibilityLevel + ,@CountCompiles AS CompileCount + ,@InitialCompileStartTime AS InitialCompileStartTime + ,@LastCompileStartTime AS LastCompileStartTime + ,@AverageCompileDuration AS AverageCompileDurationMicroseconds + ,@LastCompileDuration AS LastCompileDurationMicroseconds + ,@IsOnlineIndexPlan AS IsOnlineIndexPlan + ,@IsTrivialPlan AS IsTrivialPlan + ,@IsParallelPlan AS IsParallelPlan + ,@IsForcedPlan AS IsForcedPlan + ,@ForceFailureCount AS ForceFailureCount + ,@LastForceFailureReason AS LastForceFailureReason + ,@LastForceFailureReasonDesc AS LastForceFailureReasonDescription + ,@FirstExecutionTime AS FirstExecutionTime + ,COALESCE(@LastRuntimeExecutionTime, @LastPlanExecutionTime) AS LastExecutionTime + ,@SanitizationStatus AS SanitizationStatus + ,@SanitizationErrorCode AS SanitizationErrorCode + ,@SanitizedShowPlanXml AS SanitizedShowPlanXml + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@Start,@Rows=@Rows,@Text=@AuditText + END TRY + BEGIN CATCH + SET @CaughtErrorNumber = ERROR_NUMBER() + SET @CaughtErrorState = ERROR_STATE() + SET @AuditText = CONCAT( + N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), + N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), + N';PlanId=', ISNULL(CONVERT(nvarchar(20), @PlanId), N'NULL'), + N';SanitizationStatus=', ISNULL(CONVERT(nvarchar(32), @SanitizationStatus), N'NotCompleted'), + N';SanitizationErrorCode=PLAN_DIAGNOSTICS_FAILED', + N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), + N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)) + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@Start,@Text=@AuditText + THROW + END CATCH +END +GO + +CREATE OR ALTER PROCEDURE dbo.GetStatisticsHealth + @TableName nvarchar(128) = NULL, + @Top int = 20, + @Offset int = 0, + @OrderBy varchar(32) = 'ModificationPercent' +WITH EXECUTE AS 'dbo' +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @SP varchar(100) = OBJECT_NAME(@@PROCID); + DECLARE @Mode varchar(200) = 'StatisticsHealth'; + DECLARE @AuditText nvarchar(3500) = CONCAT( + N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), + N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), + N';TableNameSupplied=', CASE WHEN @TableName IS NULL THEN N'0' ELSE N'1' END, + N';Top=', ISNULL(CONVERT(nvarchar(11), @Top), N'NULL'), + N';Offset=', ISNULL(CONVERT(nvarchar(11), @Offset), N'NULL')); + DECLARE @Start datetime = GETUTCDATE(); + DECLARE @Rows int; + DECLARE @TableObjectId int; + DECLARE @TableCount int; + DECLARE @NormalizedOrderBy varchar(32) = UPPER(@OrderBy COLLATE Latin1_General_100_CI_AS); + DECLARE @CaughtErrorNumber int; + DECLARE @CaughtErrorState int; + + BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @AuditText; + + IF @Top IS NULL OR @Top < 1 OR @Top > 100 + BEGIN + RAISERROR('@Top must be between 1 and 100.', 18, 127); + END + + IF @Offset IS NULL OR @Offset < 0 OR @Offset > 10000 + BEGIN + RAISERROR('@Offset must be between 0 and 10000.', 18, 127); + END + + IF @NormalizedOrderBy IS NULL + OR @NormalizedOrderBy NOT IN ('MODIFICATIONCOUNT', 'MODIFICATIONPERCENT', 'LASTUPDATED', 'SAMPLINGPERCENT', 'ROWS') + BEGIN + RAISERROR('@OrderBy must be ModificationCount, ModificationPercent, LastUpdated, SamplingPercent, or Rows.', 18, 127); + END + + IF @TableName IS NOT NULL + BEGIN + IF LEN(LTRIM(RTRIM(@TableName))) = 0 + BEGIN + RAISERROR('@TableName must be nonblank when supplied.', 18, 127); + END + + SELECT + @TableCount = COUNT(*), + @TableObjectId = MIN(tableInfo.object_id) + FROM sys.tables AS tableInfo + WHERE tableInfo.is_ms_shipped = 0 + AND tableInfo.name COLLATE DATABASE_DEFAULT = @TableName COLLATE DATABASE_DEFAULT; + + IF @TableCount = 0 + BEGIN + RAISERROR('@TableName does not resolve to a user table.', 18, 127); + END + + IF @TableCount > 1 + BEGIN + RAISERROR('@TableName must resolve to exactly one user table.', 18, 127); + END + END + + SET @AuditText = CONCAT( + @AuditText, + N';TableName=', ISNULL(@TableName, N'NULL'), + N';OrderBy=', @NormalizedOrderBy); + + ;WITH StatisticsMetadata AS + ( + SELECT + tableInfo.name AS TableName, + statisticsInfo.name AS StatisticsName, + statisticsInfo.stats_id AS StatisticsId, + ( + SELECT + statisticsColumn.stats_column_id AS [@Ordinal], + columnInfo.name AS [@Name] + FROM sys.stats_columns AS statisticsColumn + INNER JOIN sys.columns AS columnInfo + ON columnInfo.object_id = statisticsColumn.object_id + AND columnInfo.column_id = statisticsColumn.column_id + WHERE statisticsColumn.object_id = statisticsInfo.object_id + AND statisticsColumn.stats_id = statisticsInfo.stats_id + ORDER BY statisticsColumn.stats_column_id + FOR XML PATH('StatisticsColumn'), ROOT('StatisticsColumns'), TYPE + ) AS StatisticsColumns, + statisticsInfo.auto_created AS AutoCreated, + statisticsInfo.user_created AS UserCreated, + statisticsInfo.is_incremental AS IsIncremental, + statisticsInfo.has_persisted_sample AS HasPersistedSample, + statisticsInfo.no_recompute AS NoRecompute, + statisticsInfo.has_filter AS HasFilter, + statisticsInfo.filter_definition AS FilterDefinition, + indexInfo.index_id AS IndexId, + indexInfo.name AS IndexName, + indexInfo.type_desc AS IndexTypeDescription, + indexInfo.is_disabled AS IsIndexDisabled, + indexInfo.is_hypothetical AS IsIndexHypothetical, + statisticsProperties.last_updated AS LastUpdated, + CONVERT(decimal(38, 4), + CONVERT(decimal(38, 0), DATEDIFF_BIG(SECOND, statisticsProperties.last_updated, SYSUTCDATETIME())) + / CONVERT(decimal(4, 0), 3600)) AS HoursSinceLastUpdate, + statisticsProperties.rows AS [Rows], + statisticsProperties.unfiltered_rows AS UnfilteredRows, + statisticsProperties.rows_sampled AS RowsSampled, + CONVERT(decimal(38, 4), + (CONVERT(decimal(38, 0), statisticsProperties.rows_sampled) * CONVERT(decimal(3, 0), 100)) + / NULLIF(CONVERT(decimal(38, 0), statisticsProperties.rows), CONVERT(decimal(38, 0), 0))) AS SamplingPercent, + statisticsProperties.steps AS HistogramStepCount, + statisticsProperties.modification_counter AS ModificationCount, + CONVERT(decimal(38, 4), + (CONVERT(decimal(38, 0), statisticsProperties.modification_counter) * CONVERT(decimal(3, 0), 100)) + / NULLIF(CONVERT(decimal(38, 0), statisticsProperties.rows), CONVERT(decimal(38, 0), 0))) AS ModificationPercent, + CASE + WHEN statisticsProperties.PropertiesAvailable IS NULL THEN 'PropertiesUnavailable' + ELSE 'Available' + END AS StatisticsStatus + FROM sys.tables AS tableInfo + INNER JOIN sys.stats AS statisticsInfo + ON statisticsInfo.object_id = tableInfo.object_id + LEFT JOIN sys.indexes AS indexInfo + ON indexInfo.object_id = statisticsInfo.object_id + AND indexInfo.index_id = statisticsInfo.stats_id + OUTER APPLY + ( + SELECT + 1 AS PropertiesAvailable, + properties.last_updated, + properties.rows, + properties.rows_sampled, + properties.steps, + properties.unfiltered_rows, + properties.modification_counter + FROM sys.dm_db_stats_properties(statisticsInfo.object_id, statisticsInfo.stats_id) AS properties + ) AS statisticsProperties + WHERE tableInfo.is_ms_shipped = 0 + AND + ( + (@TableName IS NOT NULL AND tableInfo.object_id = @TableObjectId) + OR + (@TableName IS NULL AND tableInfo.temporal_type <> 1) + ) + ), + OrderedStatisticsMetadata AS + ( + SELECT + *, + ROW_NUMBER() OVER + ( + ORDER BY + CASE WHEN @NormalizedOrderBy = 'MODIFICATIONCOUNT' THEN ModificationCount END DESC, + CASE WHEN @NormalizedOrderBy = 'MODIFICATIONPERCENT' THEN ModificationPercent END DESC, + CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' AND LastUpdated IS NULL THEN 0 + WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN 1 + END ASC, + CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN LastUpdated END ASC, + CASE WHEN @NormalizedOrderBy = 'SAMPLINGPERCENT' THEN SamplingPercent END DESC, + CASE WHEN @NormalizedOrderBy = 'ROWS' THEN [Rows] END DESC, + TableName ASC, + StatisticsName ASC, + StatisticsId ASC + ) AS RowNumber + FROM StatisticsMetadata + ) + SELECT + TableName, + StatisticsName, + StatisticsId, + StatisticsColumns, + AutoCreated, + UserCreated, + IsIncremental, + HasPersistedSample, + NoRecompute, + HasFilter, + FilterDefinition, + IndexId, + IndexName, + IndexTypeDescription, + IsIndexDisabled, + IsIndexHypothetical, + LastUpdated, + HoursSinceLastUpdate, + [Rows], + UnfilteredRows, + RowsSampled, + SamplingPercent, + HistogramStepCount, + ModificationCount, + ModificationPercent, + StatisticsStatus + FROM OrderedStatisticsMetadata + WHERE RowNumber > @Offset + AND RowNumber <= @Offset + @Top + ORDER BY RowNumber; + + SET @Rows = @@ROWCOUNT; + + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @Start, @Rows = @Rows, @Text = @AuditText; + END TRY + BEGIN CATCH + SET @CaughtErrorNumber = ERROR_NUMBER(); + SET @CaughtErrorState = ERROR_STATE(); + SET @AuditText = CONCAT( + N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), + N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), + N';StatisticsHealthErrorCode=STATISTICS_HEALTH_FAILED', + N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), + N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)); + + IF ERROR_NUMBER() = 1750 THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @Start, @Text = @AuditText; + THROW; + END CATCH +END +GO + +-- ── 3. Grant EXECUTE on each procedure individually to [FhirDiagnosticsReader] - +GRANT EXECUTE ON dbo.GetQueryStoreSlowQueries TO [FhirDiagnosticsReader]; +GRANT EXECUTE ON dbo.GetQueryStorePlanDiagnostics TO [FhirDiagnosticsReader]; +GRANT EXECUTE ON dbo.GetStatisticsHealth TO [FhirDiagnosticsReader]; +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs index 340aebb356..0642d918dc 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs @@ -126,5 +126,6 @@ public enum SchemaVersion V114 = 114, V115 = 115, V116 = 116, + V117 = 117, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index 854d345f6b..ee91e20469 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema public static class SchemaVersionConstants { public const int Min = (int)SchemaVersion.V113; - public const int Max = (int)SchemaVersion.V116; + public const int Max = (int)SchemaVersion.V117; public const int MinForUpgrade = (int)SchemaVersion.V111; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/DiagnosticsRole.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/DiagnosticsRole.sql new file mode 100644 index 0000000000..11d05a39c8 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/DiagnosticsRole.sql @@ -0,0 +1,26 @@ +IF NOT EXISTS +( + SELECT 1 + FROM sys.database_principals + WHERE name = N'FhirDiagnosticsReader' + AND type = 'R' +) +BEGIN + IF EXISTS + ( + SELECT 1 + FROM sys.database_principals + WHERE name = N'FhirDiagnosticsReader' + ) + BEGIN + THROW 50100, 'A database principal named FhirDiagnosticsReader already exists but is not a database role.', 1; + END + + CREATE ROLE [FhirDiagnosticsReader]; +END +GO + +GRANT EXECUTE ON dbo.GetQueryStoreSlowQueries TO [FhirDiagnosticsReader]; +GRANT EXECUTE ON dbo.GetQueryStorePlanDiagnostics TO [FhirDiagnosticsReader]; +GRANT EXECUTE ON dbo.GetStatisticsHealth TO [FhirDiagnosticsReader]; +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql new file mode 100644 index 0000000000..7f155e07a9 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql @@ -0,0 +1,229 @@ +--DROP PROCEDURE dbo.GetQueryStorePlanDiagnostics +GO +CREATE PROCEDURE dbo.GetQueryStorePlanDiagnostics @PlanId bigint +WITH EXECUTE AS 'dbo' +AS +BEGIN + SET NOCOUNT ON + + DECLARE @SP varchar(100) = OBJECT_NAME(@@PROCID) + ,@Mode varchar(200) = 'QueryStorePlanDiagnostics' + ,@Start datetime = GETUTCDATE() + ,@Rows int = 0 + ,@QueryStoreState nvarchar(60) + ,@QueryStoreReadonlyReason bigint + ,@AuditText nvarchar(3500) + ,@FoundPlanId bigint + ,@QueryId bigint + ,@QueryHash binary(8) + ,@QueryPlanHash binary(8) + ,@QuerySqlText nvarchar(max) + ,@ObjectId int + ,@PlanGroupId bigint + ,@EngineVersion nvarchar(128) + ,@CompatibilityLevel smallint + ,@IsOnlineIndexPlan bit + ,@IsTrivialPlan bit + ,@IsParallelPlan bit + ,@IsForcedPlan bit + ,@ForceFailureCount bigint + ,@LastForceFailureReason int + ,@LastForceFailureReasonDesc nvarchar(256) + ,@CountCompiles bigint + ,@InitialCompileStartTime datetimeoffset(7) + ,@LastCompileStartTime datetimeoffset(7) + ,@LastPlanExecutionTime datetimeoffset(7) + ,@AverageCompileDuration float + ,@LastCompileDuration bigint + ,@FirstExecutionTime datetimeoffset(7) + ,@LastRuntimeExecutionTime datetimeoffset(7) + ,@RawQueryPlan nvarchar(max) + ,@LocalPlanXml xml + ,@SanitizedShowPlanXml xml + ,@SerializedPlanXml nvarchar(max) + ,@ParameterListCount bigint + ,@ParameterListRemoved bigint = 0 + ,@RemainingParameterListCount bigint + ,@ForbiddenAttributeCount bigint + ,@SanitizationStatus varchar(32) + ,@SanitizationErrorCode varchar(64) + ,@SerializedResultSizeBytes bigint = 0 + ,@CaughtErrorNumber int + ,@CaughtErrorState int + + SET @AuditText = CONCAT( + N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), + N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), + N';PlanId=', ISNULL(CONVERT(nvarchar(20), @PlanId), N'NULL')) + + BEGIN TRY + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Start',@Text=@AuditText + + IF @PlanId IS NULL OR @PlanId <= 0 + THROW 50001, 'Plan ID must be a positive bigint.', 1 + + SELECT @QueryStoreState = actual_state_desc + ,@QueryStoreReadonlyReason = readonly_reason + FROM sys.database_query_store_options + + SET @AuditText = CONCAT( + @AuditText, + N';QueryStoreState=', ISNULL(CONVERT(nvarchar(60), @QueryStoreState), N'Unknown'), + N';QueryStoreReadonlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadonlyReason), N'NULL')) + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Run',@Text=@AuditText + + IF @QueryStoreState IS NULL OR @QueryStoreState NOT IN ('READ_WRITE', 'READ_ONLY') + THROW 50002, 'Query Store is not readable.', 1 + + SELECT @FoundPlanId = p.plan_id + ,@QueryId = p.query_id + ,@QueryHash = q.query_hash + ,@QueryPlanHash = p.query_plan_hash + ,@QuerySqlText = qt.query_sql_text + ,@ObjectId = q.object_id + ,@PlanGroupId = p.plan_group_id + ,@EngineVersion = p.engine_version + ,@CompatibilityLevel = p.compatibility_level + ,@IsOnlineIndexPlan = p.is_online_index_plan + ,@IsTrivialPlan = p.is_trivial_plan + ,@IsParallelPlan = p.is_parallel_plan + ,@IsForcedPlan = p.is_forced_plan + ,@ForceFailureCount = p.force_failure_count + ,@LastForceFailureReason = p.last_force_failure_reason + ,@LastForceFailureReasonDesc = p.last_force_failure_reason_desc + ,@CountCompiles = p.count_compiles + ,@InitialCompileStartTime = p.initial_compile_start_time + ,@LastCompileStartTime = p.last_compile_start_time + ,@LastPlanExecutionTime = p.last_execution_time + ,@AverageCompileDuration = p.avg_compile_duration + ,@LastCompileDuration = p.last_compile_duration + ,@RawQueryPlan = CONVERT(nvarchar(max), p.query_plan) + FROM sys.query_store_plan AS p + LEFT JOIN sys.query_store_query AS q + ON q.query_id = p.query_id + LEFT JOIN sys.query_store_query_text AS qt + ON qt.query_text_id = q.query_text_id + WHERE p.plan_id = @PlanId + + IF @FoundPlanId IS NULL + THROW 50003, 'The requested Query Store plan was not found or is no longer retained.', 1 + + SELECT @FirstExecutionTime = MIN(first_execution_time) + ,@LastRuntimeExecutionTime = MAX(last_execution_time) + FROM sys.query_store_runtime_stats + WHERE plan_id = @PlanId + + IF @RawQueryPlan IS NULL + BEGIN + SET @SanitizationStatus = 'PlanXmlUnavailable' + SET @SanitizationErrorCode = 'PLAN_XML_UNAVAILABLE' + END + ELSE + BEGIN + SET @LocalPlanXml = TRY_CONVERT(xml, @RawQueryPlan) + + IF @LocalPlanXml IS NULL + BEGIN + SET @SanitizationStatus = 'InvalidXml' + SET @SanitizationErrorCode = 'PLAN_XML_INVALID' + END + ELSE + BEGIN + BEGIN TRY + SET @SanitizedShowPlanXml = @LocalPlanXml + SET @ParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') + + WHILE @ParameterListRemoved < @ParameterListCount + BEGIN + SET @SanitizedShowPlanXml.modify('delete (//*[local-name(.) = "ParameterList"])[1]') + SET @ParameterListRemoved = @ParameterListRemoved + 1 + END + + SET @RemainingParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') + SET @ForbiddenAttributeCount = @SanitizedShowPlanXml.value('count(//@*[local-name(.) = "ParameterCompiledValue" or local-name(.) = "ParameterRuntimeValue"])', 'bigint') + SET @SerializedPlanXml = CONVERT(nvarchar(max), @SanitizedShowPlanXml) + + IF @RemainingParameterListCount = 0 + AND @ForbiddenAttributeCount = 0 + AND CHARINDEX(N'PARAMETERLIST', UPPER(@SerializedPlanXml)) = 0 + AND CHARINDEX(N'PARAMETERCOMPILEDVALUE', UPPER(@SerializedPlanXml)) = 0 + AND CHARINDEX(N'PARAMETERRUNTIMEVALUE', UPPER(@SerializedPlanXml)) = 0 + BEGIN + SET @SanitizationStatus = 'Sanitized' + END + ELSE + BEGIN + SET @SanitizedShowPlanXml = NULL + SET @SerializedPlanXml = NULL + SET @SanitizationStatus = 'VerificationFailed' + SET @SanitizationErrorCode = 'PLAN_XML_VERIFICATION_FAILED' + END + END TRY + BEGIN CATCH + SET @SanitizedShowPlanXml = NULL + SET @SerializedPlanXml = NULL + SET @SanitizationStatus = 'VerificationFailed' + SET @SanitizationErrorCode = 'PLAN_XML_VERIFICATION_FAILED' + END CATCH + END + END + + SET @SerializedResultSizeBytes = ISNULL(DATALENGTH(@SerializedPlanXml), 0) + SET @AuditText = CONCAT( + @AuditText, + N';SanitizationStatus=', ISNULL(CONVERT(nvarchar(32), @SanitizationStatus), N'NotStarted'), + N';SanitizationErrorCode=', ISNULL(CONVERT(nvarchar(64), @SanitizationErrorCode), N'NONE'), + N';SerializedResultSizeBytes=', CONVERT(nvarchar(20), @SerializedResultSizeBytes)) + + IF @SanitizationErrorCode IS NOT NULL + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Text=@AuditText + + SET @Rows = 1 + + SELECT @FoundPlanId AS PlanId + ,@QueryId AS QueryId + ,@QueryHash AS QueryHash + ,@QueryPlanHash AS QueryPlanHash + ,@QuerySqlText AS QuerySqlText + ,@ObjectId AS ObjectId + ,@PlanGroupId AS PlanGroupId + ,@EngineVersion AS EngineVersion + ,@CompatibilityLevel AS CompatibilityLevel + ,@CountCompiles AS CompileCount + ,@InitialCompileStartTime AS InitialCompileStartTime + ,@LastCompileStartTime AS LastCompileStartTime + ,@AverageCompileDuration AS AverageCompileDurationMicroseconds + ,@LastCompileDuration AS LastCompileDurationMicroseconds + ,@IsOnlineIndexPlan AS IsOnlineIndexPlan + ,@IsTrivialPlan AS IsTrivialPlan + ,@IsParallelPlan AS IsParallelPlan + ,@IsForcedPlan AS IsForcedPlan + ,@ForceFailureCount AS ForceFailureCount + ,@LastForceFailureReason AS LastForceFailureReason + ,@LastForceFailureReasonDesc AS LastForceFailureReasonDescription + ,@FirstExecutionTime AS FirstExecutionTime + ,COALESCE(@LastRuntimeExecutionTime, @LastPlanExecutionTime) AS LastExecutionTime + ,@SanitizationStatus AS SanitizationStatus + ,@SanitizationErrorCode AS SanitizationErrorCode + ,@SanitizedShowPlanXml AS SanitizedShowPlanXml + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@Start,@Rows=@Rows,@Text=@AuditText + END TRY + BEGIN CATCH + SET @CaughtErrorNumber = ERROR_NUMBER() + SET @CaughtErrorState = ERROR_STATE() + SET @AuditText = CONCAT( + N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), + N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), + N';PlanId=', ISNULL(CONVERT(nvarchar(20), @PlanId), N'NULL'), + N';SanitizationStatus=', ISNULL(CONVERT(nvarchar(32), @SanitizationStatus), N'NotCompleted'), + N';SanitizationErrorCode=PLAN_DIAGNOSTICS_FAILED', + N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), + N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)) + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@Start,@Text=@AuditText + THROW + END CATCH +END +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql new file mode 100644 index 0000000000..08c583d6c6 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql @@ -0,0 +1,392 @@ +--DROP PROCEDURE dbo.GetQueryStoreSlowQueries +GO +CREATE PROCEDURE dbo.GetQueryStoreSlowQueries + @StartTime datetimeoffset(7) = NULL + ,@EndTime datetimeoffset(7) = NULL + ,@Top int = 20 + ,@Offset int = 0 + ,@OrderBy varchar(32) = 'TotalDuration' + ,@MinExecutions bigint = 1 + ,@QueryTextContains nvarchar(256) = NULL +WITH EXECUTE AS 'dbo' +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @ProcedureName varchar(100) = OBJECT_NAME(@@PROCID); + DECLARE @AuditMode varchar(200) = 'QueryStoreSlowQueries'; + DECLARE @AuditStartTime datetime = GETUTCDATE(); + DECLARE @AuditText nvarchar(3500); + DECLARE @RowsReturned bigint; + DECLARE @ResolvedStartTime datetimeoffset(7); + DECLARE @ResolvedEndTime datetimeoffset(7); + DECLARE @OrderByNormalized varchar(32); + DECLARE @QueryTextPattern nvarchar(514); + DECLARE @QueryTextFilterLength int; + DECLARE @QueryStoreState nvarchar(60); + DECLARE @QueryStoreReadOnlyReason bigint; + DECLARE @WaitStatsCaptureMode nvarchar(60); + DECLARE @WaitStatsStatus varchar(32); + DECLARE @SlowQueriesProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStoreSlowQueries'); + DECLARE @PlanDiagnosticsProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStorePlanDiagnostics'); + DECLARE @StatisticsHealthProcedureObjectId int = OBJECT_ID(N'dbo.GetStatisticsHealth'); + + IF @ProcedureName IS NULL + SET @ProcedureName = 'GetQueryStoreSlowQueries'; + + SET @AuditText = CONCAT( + N'OriginalLogin=', ORIGINAL_LOGIN(), + N';EffectivePrincipal=', USER_NAME()); + + BEGIN TRY + SET @Top = ISNULL(@Top, 20); + SET @Offset = ISNULL(@Offset, 0); + SET @MinExecutions = ISNULL(@MinExecutions, 1); + SET @OrderBy = ISNULL(@OrderBy, 'TotalDuration'); + SET @ResolvedEndTime = SWITCHOFFSET(ISNULL(@EndTime, TODATETIMEOFFSET(SYSUTCDATETIME(), '+00:00')), '+00:00'); + SET @ResolvedStartTime = SWITCHOFFSET(ISNULL(@StartTime, DATEADD(hour, -1, @ResolvedEndTime)), '+00:00'); + SET @QueryTextContains = NULLIF(LTRIM(RTRIM(@QueryTextContains)), N''); + SET @QueryTextFilterLength = ISNULL(LEN(@QueryTextContains), 0); + + SELECT + @QueryStoreState = actual_state_desc, + @QueryStoreReadOnlyReason = readonly_reason, + @WaitStatsCaptureMode = wait_stats_capture_mode_desc + FROM sys.database_query_store_options; + + SET @WaitStatsStatus = + CASE @WaitStatsCaptureMode + WHEN N'ON' THEN 'Available' + WHEN N'OFF' THEN 'Disabled' + ELSE 'Unavailable' + END; + SET @AuditText = CONCAT( + @AuditText, + N';StartTimeUtc=', CONVERT(nvarchar(33), @ResolvedStartTime, 127), + N';EndTimeUtc=', CONVERT(nvarchar(33), @ResolvedEndTime, 127), + N';OrderBy=', @OrderBy, + N';Top=', @Top, + N';Offset=', @Offset, + N';MinExecutions=', @MinExecutions, + N';QueryTextFilterPresent=', CASE WHEN @QueryTextContains IS NULL THEN N'0' ELSE N'1' END, + N';QueryTextFilterLength=', @QueryTextFilterLength, + N';QueryStoreState=', ISNULL(@QueryStoreState, N'Unknown'), + N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown'), + N';WaitStatsCaptureMode=', ISNULL(@WaitStatsCaptureMode, N'Unknown')); + + EXECUTE dbo.LogEvent + @Process = @ProcedureName, + @Mode = @AuditMode, + @Status = 'Start', + @Text = @AuditText; + + IF @Top < 1 OR @Top > 100 + THROW 50400, '@Top must be between 1 and 100.', 1; + + IF @Offset < 0 OR @Offset > 10000 + THROW 50401, '@Offset must be between 0 and 10000.', 1; + + IF @MinExecutions < 1 + THROW 50402, '@MinExecutions must be positive.', 1; + + IF @ResolvedStartTime >= @ResolvedEndTime + THROW 50403, '@StartTime must precede @EndTime.', 1; + + IF @ResolvedEndTime > DATEADD(hour, 24, @ResolvedStartTime) + THROW 50404, 'The requested time range must not exceed 24 hours.', 1; + + IF @QueryTextContains IS NOT NULL + AND LEN(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(@QueryTextContains, N' ', N''), NCHAR(9), N''), NCHAR(10), N''), NCHAR(13), N''), NCHAR(160), N'')) = 0 + THROW 50405, '@QueryTextContains must not be whitespace only.', 1; + + IF @QueryTextContains IS NOT NULL + AND (@QueryTextFilterLength < 3 OR @QueryTextFilterLength > 256) + THROW 50406, '@QueryTextContains must contain between 3 and 256 characters after trimming.', 1; + + SET @OrderByNormalized = + CASE + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalDuration' THEN 'TotalDuration' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageDuration' THEN 'AverageDuration' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'MaximumDuration' THEN 'MaximumDuration' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalCpu' THEN 'TotalCpu' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageCpu' THEN 'AverageCpu' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'LogicalReads' THEN 'LogicalReads' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'Executions' THEN 'Executions' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalWait' THEN 'TotalWait' + END; + + IF @OrderByNormalized IS NULL + THROW 50407, '@OrderBy is not supported.', 1; + + IF ISNULL(@QueryStoreState, N'') NOT IN (N'READ_WRITE', N'READ_ONLY') + THROW 50408, 'Query Store is not enabled and readable.', 1; + + SET @QueryTextPattern = + CASE + WHEN @QueryTextContains IS NULL THEN NULL + ELSE N'%' + REPLACE(REPLACE(REPLACE(REPLACE(@QueryTextContains, N'~', N'~~'), N'%', N'~%'), N'_', N'~_'), N'[', N'~[') + N'%' + END; + + ;WITH RuntimeStatsRows AS + ( + SELECT + rs.plan_id AS PlanId, + rs.execution_type AS ExecutionType, + rs.runtime_stats_interval_id AS RuntimeStatsIntervalId, + rs.runtime_stats_id AS RuntimeStatsId, + rs.count_executions AS RegularExecutionCount, + CONVERT(decimal(38, 4), rs.avg_duration) AS AverageDurationMicroseconds, + CONVERT(decimal(38, 0), rs.min_duration) AS MinimumDurationMicroseconds, + CONVERT(decimal(38, 0), rs.max_duration) AS MaximumDurationMicroseconds, + CONVERT(decimal(38, 0), rs.last_duration) AS LastDurationMicroseconds, + CONVERT(decimal(38, 4), rs.avg_cpu_time) AS AverageCpuMicroseconds, + CONVERT(decimal(38, 4), rs.avg_logical_io_reads) AS AverageLogicalReads, + CONVERT(decimal(38, 4), rs.avg_physical_io_reads) AS AveragePhysicalReads, + CONVERT(decimal(38, 4), rs.avg_logical_io_writes) AS AverageLogicalWrites, + CONVERT(decimal(38, 4), rs.avg_rowcount) AS AverageRowCount, + CONVERT(decimal(38, 0), rs.max_rowcount) AS MaximumRowCount, + SWITCHOFFSET(rs.first_execution_time, '+00:00') AS FirstExecutionTimeUtc, + SWITCHOFFSET(rs.last_execution_time, '+00:00') AS LastExecutionTimeUtc + FROM sys.query_store_runtime_stats AS rs + INNER JOIN sys.query_store_runtime_stats_interval AS rsi + ON rsi.runtime_stats_interval_id = rs.runtime_stats_interval_id + -- The baseline is regular executions only. An explicit execution-type input belongs here if added later. + WHERE rs.execution_type = 0 + AND rsi.start_time < @ResolvedEndTime + AND rsi.end_time > @ResolvedStartTime + ), + RankedRuntimeStatsRows AS + ( + SELECT + rs.*, + ROW_NUMBER() OVER + ( + PARTITION BY rs.PlanId, rs.ExecutionType, rs.RuntimeStatsIntervalId + ORDER BY rs.LastExecutionTimeUtc DESC, rs.RuntimeStatsIntervalId DESC, rs.RuntimeStatsId DESC + ) AS LastValueRank + FROM RuntimeStatsRows AS rs + ), + CollapsedRuntimeStats AS + ( + SELECT + rs.PlanId, + rs.ExecutionType, + rs.RuntimeStatsIntervalId, + SUM(rs.RegularExecutionCount) AS RegularExecutionCount, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageDurationMicroseconds * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalDurationMicroseconds, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageCpuMicroseconds * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalCpuMicroseconds, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageLogicalReads * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalLogicalReads, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AveragePhysicalReads * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalPhysicalReads, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageLogicalWrites * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalLogicalWrites, + CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageRowCount * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalRowCount, + MIN(rs.MinimumDurationMicroseconds) AS MinimumDurationMicroseconds, + MAX(rs.MaximumDurationMicroseconds) AS MaximumDurationMicroseconds, + MAX(rs.MaximumRowCount) AS MaximumRowCount, + MIN(rs.FirstExecutionTimeUtc) AS FirstExecutionTimeUtc, + MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastExecutionTimeUtc END) AS LastExecutionTimeUtc, + MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastDurationMicroseconds END) AS LastDurationMicroseconds, + MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.RuntimeStatsId END) AS LastRuntimeStatsId + FROM RankedRuntimeStatsRows AS rs + GROUP BY + rs.PlanId, + rs.ExecutionType, + rs.RuntimeStatsIntervalId + ), + RankedCollapsedRuntimeStats AS + ( + SELECT + rs.*, + ROW_NUMBER() OVER + ( + PARTITION BY rs.PlanId + ORDER BY rs.LastExecutionTimeUtc DESC, rs.RuntimeStatsIntervalId DESC, rs.LastRuntimeStatsId DESC + ) AS LastValueRank + FROM CollapsedRuntimeStats AS rs + ), + AggregatedRuntimeStats AS + ( + SELECT + rs.PlanId, + SUM(rs.RegularExecutionCount) AS RegularExecutionCount, + CONVERT(decimal(38, 0), SUM(rs.TotalDurationMicroseconds)) AS TotalDurationMicroseconds, + CONVERT(decimal(38, 4), SUM(rs.TotalDurationMicroseconds) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageDurationMicroseconds, + MIN(rs.MinimumDurationMicroseconds) AS MinimumDurationMicroseconds, + MAX(rs.MaximumDurationMicroseconds) AS MaximumDurationMicroseconds, + MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastDurationMicroseconds END) AS LastDurationMicroseconds, + CONVERT(decimal(38, 0), SUM(rs.TotalCpuMicroseconds)) AS TotalCpuMicroseconds, + CONVERT(decimal(38, 4), SUM(rs.TotalCpuMicroseconds) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageCpuMicroseconds, + CONVERT(decimal(38, 0), SUM(rs.TotalLogicalReads)) AS TotalLogicalReads, + CONVERT(decimal(38, 4), SUM(rs.TotalLogicalReads) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageLogicalReads, + CONVERT(decimal(38, 0), SUM(rs.TotalPhysicalReads)) AS TotalPhysicalReads, + CONVERT(decimal(38, 4), SUM(rs.TotalPhysicalReads) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AveragePhysicalReads, + CONVERT(decimal(38, 0), SUM(rs.TotalLogicalWrites)) AS TotalLogicalWrites, + CONVERT(decimal(38, 4), SUM(rs.TotalLogicalWrites) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageLogicalWrites, + CONVERT(decimal(38, 4), SUM(rs.TotalRowCount) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageRowCount, + MAX(rs.MaximumRowCount) AS MaximumRowCount, + MIN(rs.FirstExecutionTimeUtc) AS FirstExecutionTimeUtc, + MAX(rs.LastExecutionTimeUtc) AS LastExecutionTimeUtc + FROM RankedCollapsedRuntimeStats AS rs + GROUP BY rs.PlanId + HAVING SUM(rs.RegularExecutionCount) >= @MinExecutions + ), + WaitStatsRows AS + ( + SELECT + ws.plan_id AS PlanId, + ws.wait_category_desc AS WaitCategoryDescription, + CONVERT(decimal(38, 0), ws.total_query_wait_time_ms) AS TotalWaitMilliseconds, + CONVERT(decimal(38, 0), ws.max_query_wait_time_ms) AS MaximumWaitMilliseconds + FROM sys.query_store_wait_stats AS ws + INNER JOIN sys.query_store_runtime_stats_interval AS rsi + ON rsi.runtime_stats_interval_id = ws.runtime_stats_interval_id + WHERE @WaitStatsStatus = 'Available' + AND ws.execution_type = 0 + AND rsi.start_time < @ResolvedEndTime + AND rsi.end_time > @ResolvedStartTime + ), + WaitCategories AS + ( + SELECT + ws.PlanId, + ws.WaitCategoryDescription, + CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds, + MAX(ws.MaximumWaitMilliseconds) AS MaximumWaitMilliseconds + FROM WaitStatsRows AS ws + GROUP BY + ws.PlanId, + ws.WaitCategoryDescription + ), + AggregatedWaitStats AS + ( + SELECT + ws.PlanId, + CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds + FROM WaitCategories AS ws + GROUP BY ws.PlanId + ), + WaitStatsXml AS + ( + SELECT + wp.PlanId, + ( + SELECT + wc.WaitCategoryDescription AS [@Category], + wc.TotalWaitMilliseconds AS [@TotalWaitMilliseconds], + CONVERT(decimal(38, 4), wc.TotalWaitMilliseconds / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) AS [@AverageWaitMilliseconds], + wc.MaximumWaitMilliseconds AS [@MaximumWaitMilliseconds] + FROM WaitCategories AS wc + WHERE wc.PlanId = wp.PlanId + AND wc.TotalWaitMilliseconds > 0 + ORDER BY + wc.TotalWaitMilliseconds DESC, + wc.WaitCategoryDescription ASC + FOR XML PATH(N'WaitCategory'), ROOT(N'WaitStats'), TYPE + ) AS WaitStatsXml + FROM (SELECT DISTINCT PlanId FROM WaitCategories) AS wp + INNER JOIN AggregatedRuntimeStats AS ars + ON ars.PlanId = wp.PlanId + ) + SELECT + q.query_id AS QueryId, + p.plan_id AS PlanId, + q.query_hash AS QueryHash, + p.query_plan_hash AS QueryPlanHash, + qt.query_sql_text AS QuerySqlText, + q.object_id AS ObjectId, + OBJECT_NAME(q.object_id) AS ObjectName, + ars.RegularExecutionCount, + ars.TotalDurationMicroseconds, + ars.AverageDurationMicroseconds, + ars.MinimumDurationMicroseconds, + ars.MaximumDurationMicroseconds, + ars.LastDurationMicroseconds, + ars.TotalCpuMicroseconds, + ars.AverageCpuMicroseconds, + ars.TotalLogicalReads, + ars.AverageLogicalReads, + ars.TotalPhysicalReads, + ars.AveragePhysicalReads, + ars.TotalLogicalWrites, + ars.AverageLogicalWrites, + ars.AverageRowCount, + ars.MaximumRowCount, + ars.FirstExecutionTimeUtc, + ars.LastExecutionTimeUtc, + q.count_compiles AS QueryLevelCompileCount, + SWITCHOFFSET(q.last_compile_start_time, '+00:00') AS QueryLevelLastCompileTimeUtc, + p.is_forced_plan AS IsForcedPlan, + p.force_failure_count AS ForceFailureCount, + p.last_force_failure_reason AS LastForceFailureReason, + p.last_force_failure_reason_desc AS LastForceFailureReasonDescription, + p.plan_group_id AS PlanGroupId, + p.engine_version AS EngineVersion, + p.compatibility_level AS CompatibilityLevel, + p.is_online_index_plan AS IsOnlineIndexPlan, + p.is_trivial_plan AS IsTrivialPlan, + p.is_parallel_plan AS IsParallelPlan, + CASE + WHEN @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) + END AS TotalWaitMilliseconds, + CASE + WHEN @WaitStatsStatus = 'Available' THEN CONVERT(decimal(38, 4), ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) + END AS AverageWaitMilliseconds, + @WaitStatsStatus AS WaitStatsStatus, + CASE + WHEN @WaitStatsStatus = 'Available' THEN ISNULL(wsx.WaitStatsXml, CONVERT(xml, N'')) + END AS WaitStatsXml + FROM AggregatedRuntimeStats AS ars + INNER JOIN sys.query_store_plan AS p + ON p.plan_id = ars.PlanId + INNER JOIN sys.query_store_query AS q + ON q.query_id = p.query_id + INNER JOIN sys.query_store_query_text AS qt + ON qt.query_text_id = q.query_text_id + LEFT JOIN AggregatedWaitStats AS aws + ON aws.PlanId = ars.PlanId + LEFT JOIN WaitStatsXml AS wsx + ON wsx.PlanId = ars.PlanId + WHERE ISNULL(q.object_id, -1) <> ISNULL(@SlowQueriesProcedureObjectId, -2) + AND ISNULL(q.object_id, -1) <> ISNULL(@PlanDiagnosticsProcedureObjectId, -2) + AND ISNULL(q.object_id, -1) <> ISNULL(@StatisticsHealthProcedureObjectId, -2) + AND (@QueryTextContains IS NULL OR qt.query_sql_text LIKE @QueryTextPattern ESCAPE N'~') + ORDER BY + CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' AND aws.TotalWaitMilliseconds IS NULL THEN 1 ELSE 0 END ASC, + CASE WHEN @OrderByNormalized = 'TotalDuration' THEN ars.TotalDurationMicroseconds END DESC, + CASE WHEN @OrderByNormalized = 'AverageDuration' THEN ars.AverageDurationMicroseconds END DESC, + CASE WHEN @OrderByNormalized = 'MaximumDuration' THEN ars.MaximumDurationMicroseconds END DESC, + CASE WHEN @OrderByNormalized = 'TotalCpu' THEN ars.TotalCpuMicroseconds END DESC, + CASE WHEN @OrderByNormalized = 'AverageCpu' THEN ars.AverageCpuMicroseconds END DESC, + CASE WHEN @OrderByNormalized = 'LogicalReads' THEN ars.TotalLogicalReads END DESC, + CASE WHEN @OrderByNormalized = 'Executions' THEN ars.RegularExecutionCount END DESC, + CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) END DESC, + q.query_id ASC, + p.plan_id ASC + OFFSET @Offset ROWS FETCH NEXT @Top ROWS ONLY; + + SET @RowsReturned = @@ROWCOUNT; + + EXECUTE dbo.LogEvent + @Process = @ProcedureName, + @Mode = @AuditMode, + @Status = 'End', + @Rows = @RowsReturned, + @Start = @AuditStartTime, + @Text = @AuditText; + END TRY + BEGIN CATCH + SET @AuditText = CONCAT( + @AuditText, + N';ErrorNumber=', ERROR_NUMBER(), + N';ErrorState=', ERROR_STATE()); + + EXECUTE dbo.LogEvent + @Process = @ProcedureName, + @Mode = @AuditMode, + @Status = 'Error', + @Start = @AuditStartTime, + @Text = @AuditText; + + THROW; + END CATCH +END +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql new file mode 100644 index 0000000000..d6b18554a9 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql @@ -0,0 +1,224 @@ +CREATE OR ALTER PROCEDURE dbo.GetStatisticsHealth + @TableName nvarchar(128) = NULL, + @Top int = 20, + @Offset int = 0, + @OrderBy varchar(32) = 'ModificationPercent' +WITH EXECUTE AS 'dbo' +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @SP varchar(100) = OBJECT_NAME(@@PROCID); + DECLARE @Mode varchar(200) = 'StatisticsHealth'; + DECLARE @AuditText nvarchar(3500) = CONCAT( + N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), + N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), + N';TableNameSupplied=', CASE WHEN @TableName IS NULL THEN N'0' ELSE N'1' END, + N';Top=', ISNULL(CONVERT(nvarchar(11), @Top), N'NULL'), + N';Offset=', ISNULL(CONVERT(nvarchar(11), @Offset), N'NULL')); + DECLARE @Start datetime = GETUTCDATE(); + DECLARE @Rows int; + DECLARE @TableObjectId int; + DECLARE @TableCount int; + DECLARE @NormalizedOrderBy varchar(32) = UPPER(@OrderBy COLLATE Latin1_General_100_CI_AS); + DECLARE @CaughtErrorNumber int; + DECLARE @CaughtErrorState int; + + BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @AuditText; + + IF @Top IS NULL OR @Top < 1 OR @Top > 100 + BEGIN + RAISERROR('@Top must be between 1 and 100.', 18, 127); + END + + IF @Offset IS NULL OR @Offset < 0 OR @Offset > 10000 + BEGIN + RAISERROR('@Offset must be between 0 and 10000.', 18, 127); + END + + IF @NormalizedOrderBy IS NULL + OR @NormalizedOrderBy NOT IN ('MODIFICATIONCOUNT', 'MODIFICATIONPERCENT', 'LASTUPDATED', 'SAMPLINGPERCENT', 'ROWS') + BEGIN + RAISERROR('@OrderBy must be ModificationCount, ModificationPercent, LastUpdated, SamplingPercent, or Rows.', 18, 127); + END + + IF @TableName IS NOT NULL + BEGIN + IF LEN(LTRIM(RTRIM(@TableName))) = 0 + BEGIN + RAISERROR('@TableName must be nonblank when supplied.', 18, 127); + END + + SELECT + @TableCount = COUNT(*), + @TableObjectId = MIN(tableInfo.object_id) + FROM sys.tables AS tableInfo + WHERE tableInfo.is_ms_shipped = 0 + AND tableInfo.name COLLATE DATABASE_DEFAULT = @TableName COLLATE DATABASE_DEFAULT; + + IF @TableCount = 0 + BEGIN + RAISERROR('@TableName does not resolve to a user table.', 18, 127); + END + + IF @TableCount > 1 + BEGIN + RAISERROR('@TableName must resolve to exactly one user table.', 18, 127); + END + END + + SET @AuditText = CONCAT( + @AuditText, + N';TableName=', ISNULL(@TableName, N'NULL'), + N';OrderBy=', @NormalizedOrderBy); + + ;WITH StatisticsMetadata AS + ( + SELECT + tableInfo.name AS TableName, + statisticsInfo.name AS StatisticsName, + statisticsInfo.stats_id AS StatisticsId, + ( + SELECT + statisticsColumn.stats_column_id AS [@Ordinal], + columnInfo.name AS [@Name] + FROM sys.stats_columns AS statisticsColumn + INNER JOIN sys.columns AS columnInfo + ON columnInfo.object_id = statisticsColumn.object_id + AND columnInfo.column_id = statisticsColumn.column_id + WHERE statisticsColumn.object_id = statisticsInfo.object_id + AND statisticsColumn.stats_id = statisticsInfo.stats_id + ORDER BY statisticsColumn.stats_column_id + FOR XML PATH('StatisticsColumn'), ROOT('StatisticsColumns'), TYPE + ) AS StatisticsColumns, + statisticsInfo.auto_created AS AutoCreated, + statisticsInfo.user_created AS UserCreated, + statisticsInfo.is_incremental AS IsIncremental, + statisticsInfo.has_persisted_sample AS HasPersistedSample, + statisticsInfo.no_recompute AS NoRecompute, + statisticsInfo.has_filter AS HasFilter, + statisticsInfo.filter_definition AS FilterDefinition, + indexInfo.index_id AS IndexId, + indexInfo.name AS IndexName, + indexInfo.type_desc AS IndexTypeDescription, + indexInfo.is_disabled AS IsIndexDisabled, + indexInfo.is_hypothetical AS IsIndexHypothetical, + statisticsProperties.last_updated AS LastUpdated, + CONVERT(decimal(38, 4), + CONVERT(decimal(38, 0), DATEDIFF_BIG(SECOND, statisticsProperties.last_updated, SYSUTCDATETIME())) + / CONVERT(decimal(4, 0), 3600)) AS HoursSinceLastUpdate, + statisticsProperties.rows AS [Rows], + statisticsProperties.unfiltered_rows AS UnfilteredRows, + statisticsProperties.rows_sampled AS RowsSampled, + CONVERT(decimal(38, 4), + (CONVERT(decimal(38, 0), statisticsProperties.rows_sampled) * CONVERT(decimal(3, 0), 100)) + / NULLIF(CONVERT(decimal(38, 0), statisticsProperties.rows), CONVERT(decimal(38, 0), 0))) AS SamplingPercent, + statisticsProperties.steps AS HistogramStepCount, + statisticsProperties.modification_counter AS ModificationCount, + CONVERT(decimal(38, 4), + (CONVERT(decimal(38, 0), statisticsProperties.modification_counter) * CONVERT(decimal(3, 0), 100)) + / NULLIF(CONVERT(decimal(38, 0), statisticsProperties.rows), CONVERT(decimal(38, 0), 0))) AS ModificationPercent, + CASE + WHEN statisticsProperties.PropertiesAvailable IS NULL THEN 'PropertiesUnavailable' + ELSE 'Available' + END AS StatisticsStatus + FROM sys.tables AS tableInfo + INNER JOIN sys.stats AS statisticsInfo + ON statisticsInfo.object_id = tableInfo.object_id + LEFT JOIN sys.indexes AS indexInfo + ON indexInfo.object_id = statisticsInfo.object_id + AND indexInfo.index_id = statisticsInfo.stats_id + OUTER APPLY + ( + SELECT + 1 AS PropertiesAvailable, + properties.last_updated, + properties.rows, + properties.rows_sampled, + properties.steps, + properties.unfiltered_rows, + properties.modification_counter + FROM sys.dm_db_stats_properties(statisticsInfo.object_id, statisticsInfo.stats_id) AS properties + ) AS statisticsProperties + WHERE tableInfo.is_ms_shipped = 0 + AND + ( + (@TableName IS NOT NULL AND tableInfo.object_id = @TableObjectId) + OR + (@TableName IS NULL AND tableInfo.temporal_type <> 1) + ) + ), + OrderedStatisticsMetadata AS + ( + SELECT + *, + ROW_NUMBER() OVER + ( + ORDER BY + CASE WHEN @NormalizedOrderBy = 'MODIFICATIONCOUNT' THEN ModificationCount END DESC, + CASE WHEN @NormalizedOrderBy = 'MODIFICATIONPERCENT' THEN ModificationPercent END DESC, + CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' AND LastUpdated IS NULL THEN 0 + WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN 1 + END ASC, + CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN LastUpdated END ASC, + CASE WHEN @NormalizedOrderBy = 'SAMPLINGPERCENT' THEN SamplingPercent END DESC, + CASE WHEN @NormalizedOrderBy = 'ROWS' THEN [Rows] END DESC, + TableName ASC, + StatisticsName ASC, + StatisticsId ASC + ) AS RowNumber + FROM StatisticsMetadata + ) + SELECT + TableName, + StatisticsName, + StatisticsId, + StatisticsColumns, + AutoCreated, + UserCreated, + IsIncremental, + HasPersistedSample, + NoRecompute, + HasFilter, + FilterDefinition, + IndexId, + IndexName, + IndexTypeDescription, + IsIndexDisabled, + IsIndexHypothetical, + LastUpdated, + HoursSinceLastUpdate, + [Rows], + UnfilteredRows, + RowsSampled, + SamplingPercent, + HistogramStepCount, + ModificationCount, + ModificationPercent, + StatisticsStatus + FROM OrderedStatisticsMetadata + WHERE RowNumber > @Offset + AND RowNumber <= @Offset + @Top + ORDER BY RowNumber; + + SET @Rows = @@ROWCOUNT; + + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @Start, @Rows = @Rows, @Text = @AuditText; + END TRY + BEGIN CATCH + SET @CaughtErrorNumber = ERROR_NUMBER(); + SET @CaughtErrorState = ERROR_STATE(); + SET @AuditText = CONCAT( + N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), + N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), + N';StatisticsHealthErrorCode=STATISTICS_HEALTH_FAILED', + N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), + N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)); + + IF ERROR_NUMBER() = 1750 THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @Start, @Text = @AuditText; + THROW; + END CATCH +END +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index 9c6dfd9a2e..426273b34c 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -1,7 +1,7 @@  - 116 + 117 Features\Schema\Migrations\$(LatestSchemaVersion).sql LatestSchemaVersion-$(LatestSchemaVersion) @@ -46,6 +46,7 @@ + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems index 368f7a10da..b76140f915 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems @@ -28,6 +28,7 @@ + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs new file mode 100644 index 0000000000..5da0dd96da --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs @@ -0,0 +1,239 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Data; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Tests.Integration.Persistence +{ + /// + /// Exercises the SQL Server Query Store diagnostic procedures against captured FHIR database activity. + /// + [FhirStorageTestsFixtureArgumentSets(DataStore.SqlServer)] + public class SqlServerQueryStoreDiagnosticsTests : IClassFixture + { + private const int QueryExecutionCount = 4; + private const int QueryStorePollAttempts = 15; + private static readonly TimeSpan QueryStorePollInterval = TimeSpan.FromSeconds(1); + private readonly FhirStorageTestsFixture _fixture; + + /// + /// Initializes a new instance of the class. + /// + /// The SQL Server-backed FHIR storage fixture. + public SqlServerQueryStoreDiagnosticsTests(FhirStorageTestsFixture fixture) + { + _fixture = fixture; + } + + /// + /// Verifies the slow-query, plan diagnostics, and statistics health contracts with Query Store runtime data. + /// + /// A task that represents the asynchronous test operation. + [Fact] + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.DataSourceValidation)] + public async Task GivenAQueryStoreCapturedFhirQuery_WhenDiagnosticsProceduresAreCalled_ThenReturnSanitizedPlanAndStatisticsMetadata() + { + using SqlConnection connection = await _fixture.SqlHelper.GetSqlConnectionAsync(); + if (connection.State != ConnectionState.Open) + { + await connection.OpenAsync(CancellationToken.None); + } + + await EnableAndVerifyQueryStoreAsync(connection, CancellationToken.None); + + string queryMarker = $"QueryStoreDiagnostics{Guid.NewGuid():N}"; + DateTimeOffset windowStart = DateTimeOffset.UtcNow.AddMinutes(-5); + for (int execution = 0; execution < QueryExecutionCount; execution++) + { + using SqlCommand command = connection.CreateCommand(); + command.CommandText = $"SELECT COUNT_BIG(*) AS [{queryMarker}] FROM dbo.Resource;"; + object result = await command.ExecuteScalarAsync(CancellationToken.None); + Assert.NotNull(result); + } + + await WaitForQueryStoreCaptureAsync(connection, queryMarker, CancellationToken.None); + + long planId = await GetSlowQueryPlanIdAsync( + connection, + queryMarker, + windowStart, + DateTimeOffset.UtcNow.AddMinutes(1), + CancellationToken.None); + + await AssertPlanDiagnosticsAsync(connection, planId, queryMarker, CancellationToken.None); + await AssertResourceStatisticsHealthAsync(connection, CancellationToken.None); + } + + private static async Task EnableAndVerifyQueryStoreAsync(SqlConnection connection, CancellationToken cancellationToken) + { + string initialState = await GetQueryStoreStateAsync(connection, cancellationToken); + + if (!string.Equals(initialState, "READ_WRITE", StringComparison.OrdinalIgnoreCase)) + { + await ExecuteNonQueryAsync( + connection, + "ALTER DATABASE CURRENT SET QUERY_STORE = ON;", + cancellationToken); + } + + await ExecuteNonQueryAsync( + connection, + "ALTER DATABASE CURRENT SET QUERY_STORE (OPERATION_MODE = READ_WRITE, QUERY_CAPTURE_MODE = ALL);", + cancellationToken); + + string finalState = await GetQueryStoreStateAsync(connection, cancellationToken); + Assert.True( + string.Equals(finalState, "READ_WRITE", StringComparison.OrdinalIgnoreCase), + $"Query Store is not writable after enablement (state: {finalState ?? "unknown"})."); + } + + private static async Task GetQueryStoreStateAsync(SqlConnection connection, CancellationToken cancellationToken) + { + using SqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT actual_state_desc FROM sys.database_query_store_options;"; + return (string)await command.ExecuteScalarAsync(cancellationToken); + } + + private static async Task WaitForQueryStoreCaptureAsync(SqlConnection connection, string queryMarker, CancellationToken cancellationToken) + { + for (int attempt = 0; attempt < QueryStorePollAttempts; attempt++) + { + await ExecuteNonQueryAsync(connection, "EXEC sys.sp_query_store_flush_db;", cancellationToken); + + using SqlCommand command = connection.CreateCommand(); + command.CommandText = """ + SELECT COUNT_BIG(*) + FROM sys.query_store_query_text + WHERE query_sql_text LIKE @queryTextPattern; + """; + command.Parameters.Add("@queryTextPattern", SqlDbType.NVarChar, 256).Value = $"%{queryMarker}%"; + + long capturedQueryCount = Convert.ToInt64(await command.ExecuteScalarAsync(cancellationToken)); + if (capturedQueryCount > 0) + { + return; + } + + await Task.Delay(QueryStorePollInterval, cancellationToken); + } + + Assert.Fail("Query Store did not persist the diagnostic query after the supported flush and polling window."); + } + + private static async Task GetSlowQueryPlanIdAsync( + SqlConnection connection, + string queryMarker, + DateTimeOffset windowStart, + DateTimeOffset windowEnd, + CancellationToken cancellationToken) + { + using SqlCommand command = connection.CreateCommand(); + command.CommandType = CommandType.StoredProcedure; + command.CommandText = "dbo.GetQueryStoreSlowQueries"; + command.Parameters.Add("@StartTime", SqlDbType.DateTimeOffset).Value = windowStart; + command.Parameters.Add("@EndTime", SqlDbType.DateTimeOffset).Value = windowEnd; + command.Parameters.Add("@Top", SqlDbType.Int).Value = 10; + command.Parameters.Add("@Offset", SqlDbType.Int).Value = 0; + command.Parameters.Add("@OrderBy", SqlDbType.VarChar, 32).Value = "Executions"; + command.Parameters.Add("@MinExecutions", SqlDbType.BigInt).Value = QueryExecutionCount; + command.Parameters.Add("@QueryTextContains", SqlDbType.NVarChar, 256).Value = queryMarker; + + using SqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken); + Assert.True(await reader.ReadAsync(cancellationToken), "The uniquely marked Query Store query was not returned by dbo.GetQueryStoreSlowQueries."); + + long planId = reader.GetInt64(reader.GetOrdinal("PlanId")); + Assert.True(planId > 0); + Assert.True(reader.GetInt64(reader.GetOrdinal("QueryId")) > 0); + Assert.True(reader.GetInt64(reader.GetOrdinal("RegularExecutionCount")) >= QueryExecutionCount); + Assert.Contains(queryMarker, reader.GetString(reader.GetOrdinal("QuerySqlText"))); + Assert.False(reader.IsDBNull(reader.GetOrdinal("FirstExecutionTimeUtc"))); + Assert.False(reader.IsDBNull(reader.GetOrdinal("LastExecutionTimeUtc"))); + Assert.False(await reader.ReadAsync(cancellationToken), "The unique query-text filter returned more than one Query Store plan."); + Assert.False(await reader.NextResultAsync(cancellationToken), "dbo.GetQueryStoreSlowQueries returned more than one result set."); + + return planId; + } + + private static async Task AssertPlanDiagnosticsAsync( + SqlConnection connection, + long planId, + string queryMarker, + CancellationToken cancellationToken) + { + using SqlCommand command = connection.CreateCommand(); + command.CommandType = CommandType.StoredProcedure; + command.CommandText = "dbo.GetQueryStorePlanDiagnostics"; + command.Parameters.Add("@PlanId", SqlDbType.BigInt).Value = planId; + + using SqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken); + Assert.True(await reader.ReadAsync(cancellationToken), "dbo.GetQueryStorePlanDiagnostics did not return the selected plan."); + Assert.Equal(planId, reader.GetInt64(reader.GetOrdinal("PlanId"))); + Assert.True(reader.GetInt64(reader.GetOrdinal("QueryId")) > 0); + Assert.Contains(queryMarker, reader.GetString(reader.GetOrdinal("QuerySqlText"))); + Assert.False(reader.IsDBNull(reader.GetOrdinal("CompatibilityLevel"))); + + string sanitizationStatus = reader.GetString(reader.GetOrdinal("SanitizationStatus")); + int sanitizedShowPlanXmlOrdinal = reader.GetOrdinal("SanitizedShowPlanXml"); + + if (string.Equals(sanitizationStatus, "Sanitized", StringComparison.Ordinal)) + { + Assert.False(reader.IsDBNull(sanitizedShowPlanXmlOrdinal)); + + string sanitizedShowPlanXml = reader.GetValue(sanitizedShowPlanXmlOrdinal).ToString()!; + Assert.False(sanitizedShowPlanXml.Contains("ParameterList", StringComparison.OrdinalIgnoreCase)); + Assert.False(sanitizedShowPlanXml.Contains("ParameterCompiledValue", StringComparison.OrdinalIgnoreCase)); + Assert.False(sanitizedShowPlanXml.Contains("ParameterRuntimeValue", StringComparison.OrdinalIgnoreCase)); + } + else + { + Assert.Contains(sanitizationStatus, new[] { "PlanXmlUnavailable", "InvalidXml", "VerificationFailed" }); + Assert.True(reader.IsDBNull(sanitizedShowPlanXmlOrdinal)); + Assert.False(reader.IsDBNull(reader.GetOrdinal("SanitizationErrorCode"))); + } + + Assert.False(await reader.ReadAsync(cancellationToken), "dbo.GetQueryStorePlanDiagnostics returned more than one row."); + Assert.False(await reader.NextResultAsync(cancellationToken), "dbo.GetQueryStorePlanDiagnostics returned more than one result set."); + } + + private static async Task AssertResourceStatisticsHealthAsync(SqlConnection connection, CancellationToken cancellationToken) + { + using SqlCommand command = connection.CreateCommand(); + command.CommandType = CommandType.StoredProcedure; + command.CommandText = "dbo.GetStatisticsHealth"; + command.Parameters.Add("@TableName", SqlDbType.NVarChar, 128).Value = "Resource"; + command.Parameters.Add("@Top", SqlDbType.Int).Value = 1; + command.Parameters.Add("@Offset", SqlDbType.Int).Value = 0; + command.Parameters.Add("@OrderBy", SqlDbType.VarChar, 32).Value = "Rows"; + + using SqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken); + Assert.True(await reader.ReadAsync(cancellationToken), "dbo.GetStatisticsHealth did not return metadata for dbo.Resource."); + Assert.Equal("Resource", reader.GetString(reader.GetOrdinal("TableName"))); + Assert.False(reader.IsDBNull(reader.GetOrdinal("StatisticsName"))); + Assert.True(reader.GetInt32(reader.GetOrdinal("StatisticsId")) > 0); + Assert.False(reader.IsDBNull(reader.GetOrdinal("StatisticsColumns"))); + Assert.Contains("StatisticsColumn", reader.GetValue(reader.GetOrdinal("StatisticsColumns")).ToString()); + Assert.Contains(reader.GetString(reader.GetOrdinal("StatisticsStatus")), new[] { "Available", "PropertiesUnavailable" }); + Assert.False(await reader.ReadAsync(cancellationToken), "The bounded Resource statistics request returned more than one row."); + Assert.False(await reader.NextResultAsync(cancellationToken), "dbo.GetStatisticsHealth returned more than one result set."); + } + + private static async Task ExecuteNonQueryAsync(SqlConnection connection, string commandText, CancellationToken cancellationToken) + { + using SqlCommand command = connection.CreateCommand(); + command.CommandText = commandText; + await command.ExecuteNonQueryAsync(cancellationToken); + } + } +} From 6c0bc274a184cd9fdab0d3c4ae8df789b756fb1b Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 13 Aug 2026 22:34:41 +0000 Subject: [PATCH 04/20] Fix integration test trait placement Move the required Category and OwningTeam traits to the test class so assembly validation recognizes them.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Persistence/SqlServerQueryStoreDiagnosticsTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs index 5da0dd96da..36067d83eb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs @@ -20,6 +20,8 @@ namespace Microsoft.Health.Fhir.Tests.Integration.Persistence /// Exercises the SQL Server Query Store diagnostic procedures against captured FHIR database activity. /// [FhirStorageTestsFixtureArgumentSets(DataStore.SqlServer)] + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.DataSourceValidation)] public class SqlServerQueryStoreDiagnosticsTests : IClassFixture { private const int QueryExecutionCount = 4; @@ -41,8 +43,6 @@ public SqlServerQueryStoreDiagnosticsTests(FhirStorageTestsFixture fixture) /// /// A task that represents the asynchronous test operation. [Fact] - [Trait(Traits.OwningTeam, OwningTeam.Fhir)] - [Trait(Traits.Category, Categories.DataSourceValidation)] public async Task GivenAQueryStoreCapturedFhirQuery_WhenDiagnosticsProceduresAreCalled_ThenReturnSanitizedPlanAndStatisticsMetadata() { using SqlConnection connection = await _fixture.SqlHelper.GetSqlConnectionAsync(); From 45ddaa98c44a2d593c17e1d8014ffcd59a5a5ff3 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 13 Aug 2026 23:38:34 +0000 Subject: [PATCH 05/20] Fix plan diagnostics migration syntax Terminate the LogEvent statement before the bare THROW so incremental schema upgrades can compile GetQueryStorePlanDiagnostics.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Features/Schema/Migrations/117.diff.sql | 4 ++-- .../Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql index a421077cc2..0cc80417aa 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql @@ -633,8 +633,8 @@ BEGIN N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)) - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@Start,@Text=@AuditText - THROW + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@Start,@Text=@AuditText; + THROW; END CATCH END GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql index 7f155e07a9..5a6ae50eec 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql @@ -222,8 +222,8 @@ BEGIN N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)) - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@Start,@Text=@AuditText - THROW + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@Start,@Text=@AuditText; + THROW; END CATCH END GO From b930596998275721e051196fb62b6435b4cbc1fa Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 14 Aug 2026 00:51:57 +0000 Subject: [PATCH 06/20] Simplify Query Store diagnostics baseline Use Azure SQL diagnostic settings for wait statistics, reduce plan sanitization and audit plumbing, and keep the SQL interface focused on runtime metrics, sanitized plans, and statistics health.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 64 ++++----- .../Features/Schema/Migrations/117.diff.sql | 121 ++---------------- .../Sprocs/GetQueryStorePlanDiagnostics.sql | 19 +-- .../Sql/Sprocs/GetQueryStoreSlowQueries.sql | 90 +------------ .../Schema/Sql/Sprocs/GetStatisticsHealth.sql | 12 +- .../SqlServerQueryStoreDiagnosticsTests.cs | 22 ++++ 6 files changed, 65 insertions(+), 263 deletions(-) diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 99d3eca0f8..5050b32096 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -6,11 +6,11 @@ Agreed baseline for implementation. This document defines the SQL contract, secu ## Problem -FHIR Azure SQL performance investigations currently require privileged, manual access to Query Store plans, runtime metrics, wait statistics, and statistics metadata. Existing Log Analytics data provides some Query Store information, but support engineers still need a bounded way to: +FHIR Azure SQL performance investigations currently require privileged, manual access to Query Store plans, runtime metrics, and statistics metadata. Query Store wait statistics are supplied separately through the Azure SQL `QueryStoreWaitStatistics` diagnostic setting in Log Analytics. That diagnostic stream does not provide query text or Showplan XML. Support engineers still need a bounded way to: - identify expensive or regressed query plans; - retrieve an SSMS-viewable Query Store Showplan; -- compare runtime and wait metrics; and +- inspect runtime metrics; and - inspect statistics freshness, sampling, and cardinality metadata. The baseline is a self-contained, read-only SQL interface. Filtering, validation, redaction, paging, permissions, and auditing must live in SQL so the procedures can be used by an authorized direct SQL connection or wrapped by future operational tooling. @@ -29,15 +29,15 @@ Related internal guidance: 1. Identify slow or resource-intensive query plans over a bounded time range. 2. Return full Query Store query text to authorized diagnostic callers. 3. Return an SSMS-viewable Query Store Showplan after removing parameter-value metadata. -4. Include Query Store wait statistics in slow-query results when capture is available. -5. Report statistics freshness, sampling, and filter metadata without returning histogram values. -6. Provide an execute-only database role for least-privilege callers. -7. Keep the SQL contract independent of any API, Geneva action, or other caller implementation. +4. Report statistics freshness, sampling, and filter metadata without returning histogram values. +5. Provide an execute-only database role for least-privilege callers. +6. Keep the SQL contract independent of any API, Geneva action, or other caller implementation. ## Non-goals - Retrieving or capturing an actual execution plan. - Reconstructing or executing SQL from Query Store. +- Returning plan-level wait statistics or duplicating the Azure SQL `QueryStoreWaitStatistics` Log Analytics stream. - Returning statistics histograms, density vectors, or sampled column values. - Clearing the procedure cache, updating statistics, forcing plans, or changing Query Store configuration. - Providing caller concurrency control, circuit breaking, command timeouts, artifact retention, or download policy. @@ -50,6 +50,12 @@ Related internal guidance: The baseline targets Azure SQL Database and uses only Query Store catalog columns guaranteed across the supported Azure SQL deployment fleet at implementation time. Optional columns that may be rolling out regionally must not be referenced until they are universally available. +### Query Store wait-stat observability + +Query Store wait statistics come from the Azure SQL `QueryStoreWaitStatistics` diagnostic setting in Log Analytics and are intentionally not duplicated by this SQL interface. Log Analytics does not supply the query text or Showplan XML returned by the diagnostic procedures. + +`DatabaseWaitStatistics`, when enabled, is a separate database-level diagnostic stream. It is not plan-level data and is not a substitute for `QueryStoreWaitStatistics`. + ### Query Store plans are estimated plans `sys.query_store_plan.query_plan` contains the compile-time Showplan, equivalent to `SET SHOWPLAN_XML ON`. Query Store combines this plan with aggregated runtime statistics; it does not retain an actual plan for every execution. @@ -65,6 +71,7 @@ This is documentation only. No stub procedure, shared output contract, permissio ## Implementation simplifications - Plan-type and Parameter Sensitive Plan dispatcher/query-variant metadata are intentionally not read. Those Azure SQL catalog fields are not stable across the supported deployment fleet, so both Query Store procedures omit them rather than returning speculative NULL/status fields or attempting version-specific fallback logic. +- `GetQueryStoreSlowQueries` aggregates runtime data only. Wait analysis remains in the Azure SQL `QueryStoreWaitStatistics` Log Analytics stream; the procedure neither queries nor returns plan-level wait data. `DatabaseWaitStatistics` remains separate database-level telemetry. - `GetStatisticsHealth` reports database-level `sys.dm_db_stats_properties` metadata for each statistics object. It does not expand incremental statistics into partition-level property rows; unavailable properties remain `NULL` and are explicitly marked `PropertiesUnavailable`. Its table-name input is materialized as `nvarchar(128)`, rather than the type-equivalent `sysname` alias, because the existing schema C# model generator interprets `sysname` as a table-valued parameter. ### Accepted query and plan content @@ -118,6 +125,8 @@ The rollout order is: 4. provision approved role membership; and 5. enable the PaaS invocation and artifact-handling workflow. +Provisioning and retaining the Azure SQL `QueryStoreWaitStatistics` diagnostic setting and its Log Analytics destination is a separate observability rollout. It is not a schema prerequisite or a source for query text or Showplan XML. + ## Stored procedures All procedures: @@ -168,11 +177,10 @@ The `@OrderBy` allowlist is: - `AverageCpu` - `LogicalReads` - `Executions` -- `TotalWait` -Unknown order values fail explicitly. Ordering uses a static `CASE` expression rather than dynamic SQL. Diagnostic metrics sort descending, NULL wait totals sort last, and `query_id ASC, plan_id ASC` are deterministic tie-breakers. +Unknown order values fail explicitly. Ordering uses a static `CASE` expression rather than dynamic SQL. Diagnostic metrics sort descending, and `query_id ASC, plan_id ASC` are deterministic tie-breakers. -There is no execution-type input in the baseline. Runtime and wait metrics include regular executions only. The implementation should contain a focused comment identifying where execution-type support could be added later. +There is no execution-type input in the baseline. Runtime metrics include regular executions only. The implementation should contain a focused comment identifying where execution-type support could be added later. #### Time-window semantics @@ -203,28 +211,6 @@ Query-level compile count and last compile time are repeated on each plan row an Plans with fewer than `@MinExecutions` regular executions in the selected window are excluded. The diagnostic procedures' own Query Store entries are also excluded. All other object-bound and ad hoc Query Store entries are eligible. -#### Wait statistics - -Wait statistics are aggregated for the same regular-execution population and overlapping intervals as runtime metrics. - -Each result row contains: - -- `TotalWaitMilliseconds` -- `AverageWaitMilliseconds` -- `WaitStatsStatus` -- `WaitStatsXml` - -`WaitStatsXml` contains one element per wait category, ordered by total wait descending, with: - -- category name; -- total wait milliseconds; -- average wait milliseconds; and -- maximum wait milliseconds. - -Zero-wait categories are omitted. When wait capture is available but a plan has no waits, the value is an empty typed root such as ``. When wait capture is disabled or unavailable, `WaitStatsXml` and scalar wait metrics are NULL and `WaitStatsStatus` explains the condition. Other runtime results still return. - -If `@OrderBy = 'TotalWait'` while wait capture is disabled or unavailable, rows still return. NULL wait totals sort last. - #### Output The single result set includes: @@ -247,9 +233,6 @@ The single result set includes: - query-level compile count and last compile time - forced-plan state and available force-failure metadata - other universally available diagnostic plan metadata; plan-type, dispatcher, and query-variant metadata are omitted -- total and average wait milliseconds -- `WaitStatsStatus` -- `WaitStatsXml` Physical reads and writes are output metrics but are not ordering options. Query context/handle metadata and execution type are omitted. @@ -287,7 +270,7 @@ The raw Query Store plan must never be returned. The procedure: 1. copies `query_plan` into a local `xml` variable; 2. counts all elements whose local name is `ParameterList`, regardless of namespace; -3. removes every such element using a bounded XML DML loop; +3. removes every such element in a single XML DML operation; 4. verifies structurally that no `ParameterList` element and no `ParameterCompiledValue` or `ParameterRuntimeValue` attribute remains; 5. serializes the result and performs a case-insensitive textual check for those forbidden names; and 6. returns the XML only when every verification succeeds. @@ -433,7 +416,7 @@ SQL enforces: - a 3-256-character literal query-text substring; - one plan per plan-diagnostics call; - static SQL only; -- regular-execution-only runtime and wait aggregation; and +- regular-execution-only runtime aggregation; and - exclusion of the diagnostic procedures' own Query Store entries. Limits are hard-coded in the procedures. There is no `dbo.Parameters` kill switch, SQL concurrency gate, plan-size cap, total-count query, continuation token, or `HasMoreRows` result. @@ -471,21 +454,20 @@ If Start, End, or Error logging fails, the diagnostic call fails. Callers do not ## Testing requirements -**Deferred prototype validation:** The prototype has one representative SQL-backed end-to-end path. Exhaustive matrix validation remains future work, including negative validation, fail-closed audit behavior, permissions, a malformed-fixture corpus, and wait-disabled cases. +**Deferred prototype validation:** The prototype has one representative SQL-backed end-to-end path. Exhaustive matrix validation remains future work, including negative validation, fail-closed audit behavior, permissions, and a malformed-fixture corpus. ### Slow-query aggregation 1. Duplicate active-interval in-memory/persisted rows are collapsed before rollup. 2. Weighted totals and averages use the agreed decimal precision. 3. Minimum, maximum, and deterministic last-value calculations are correct. -4. Only regular executions contribute to runtime and wait metrics. +4. Only regular executions contribute to runtime metrics. 5. Overlapping Query Store interval semantics are verified at both window boundaries. 6. Time, row, offset, minimum-execution, query-text, and order allowlists cannot be bypassed. 7. Literal query-text matching correctly escapes `~`, `%`, `_`, and `[`. 8. Query Store `READ_WRITE` and readable `READ_ONLY` states return data. 9. Query Store `OFF`, `ERROR`, and unreadable states fail with actionable errors. -10. Wait capture available, disabled, unavailable, empty, and `TotalWait` ordering cases are covered. -11. Diagnostic procedures exclude their own Query Store entries. +10. Diagnostic procedures exclude their own Query Store entries. ### Showplan sanitization @@ -535,6 +517,7 @@ If Start, End, or Error logging fails, the diagnostic call fails. Callers do not - Update the OSS FHIR package versions and synchronized target schema version through the existing `fhir-paas` dependency flow. - Do not copy the stored procedure or role DDL into PaaS Script Runner scripts. - Add role membership only for the approved operational identity. +- Configure and operate `QueryStoreWaitStatistics` through the Azure SQL diagnostic setting and Log Analytics independently of stored-procedure deployment. - Implement the selected caller, result transport, artifact storage, and operational authorization in `fhir-paas`. - Deploy schema/package consumption before enabling the caller. - Control caller rollout and role membership independently in each environment. @@ -544,7 +527,6 @@ If Start, End, or Error logging fails, the diagnostic call fails. Callers do not - [Monitor performance by using Query Store](https://learn.microsoft.com/sql/relational-databases/performance/monitoring-performance-by-using-the-query-store) - [How Query Store collects data](https://learn.microsoft.com/sql/relational-databases/performance/how-query-store-collects-data) - [`sys.query_store_runtime_stats`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-runtime-stats-transact-sql) -- [`sys.query_store_wait_stats`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-wait-stats-transact-sql) - [`sys.query_store_plan`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-plan-transact-sql) - [`sys.query_store_query`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-query-transact-sql) - [`sys.query_store_query_text`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-query-text-transact-sql) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql index 0cc80417aa..8109bc4936 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql @@ -45,8 +45,6 @@ BEGIN DECLARE @QueryTextFilterLength int; DECLARE @QueryStoreState nvarchar(60); DECLARE @QueryStoreReadOnlyReason bigint; - DECLARE @WaitStatsCaptureMode nvarchar(60); - DECLARE @WaitStatsStatus varchar(32); DECLARE @SlowQueriesProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStoreSlowQueries'); DECLARE @PlanDiagnosticsProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStorePlanDiagnostics'); DECLARE @StatisticsHealthProcedureObjectId int = OBJECT_ID(N'dbo.GetStatisticsHealth'); @@ -70,16 +68,9 @@ BEGIN SELECT @QueryStoreState = actual_state_desc, - @QueryStoreReadOnlyReason = readonly_reason, - @WaitStatsCaptureMode = wait_stats_capture_mode_desc + @QueryStoreReadOnlyReason = readonly_reason FROM sys.database_query_store_options; - SET @WaitStatsStatus = - CASE @WaitStatsCaptureMode - WHEN N'ON' THEN 'Available' - WHEN N'OFF' THEN 'Disabled' - ELSE 'Unavailable' - END; SET @AuditText = CONCAT( @AuditText, N';StartTimeUtc=', CONVERT(nvarchar(33), @ResolvedStartTime, 127), @@ -91,8 +82,7 @@ BEGIN N';QueryTextFilterPresent=', CASE WHEN @QueryTextContains IS NULL THEN N'0' ELSE N'1' END, N';QueryTextFilterLength=', @QueryTextFilterLength, N';QueryStoreState=', ISNULL(@QueryStoreState, N'Unknown'), - N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown'), - N';WaitStatsCaptureMode=', ISNULL(@WaitStatsCaptureMode, N'Unknown')); + N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown')); EXECUTE dbo.LogEvent @Process = @ProcedureName, @@ -132,7 +122,6 @@ BEGIN WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageCpu' THEN 'AverageCpu' WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'LogicalReads' THEN 'LogicalReads' WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'Executions' THEN 'Executions' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalWait' THEN 'TotalWait' END; IF @OrderByNormalized IS NULL @@ -248,63 +237,6 @@ BEGIN FROM RankedCollapsedRuntimeStats AS rs GROUP BY rs.PlanId HAVING SUM(rs.RegularExecutionCount) >= @MinExecutions - ), - WaitStatsRows AS - ( - SELECT - ws.plan_id AS PlanId, - ws.wait_category_desc AS WaitCategoryDescription, - CONVERT(decimal(38, 0), ws.total_query_wait_time_ms) AS TotalWaitMilliseconds, - CONVERT(decimal(38, 0), ws.max_query_wait_time_ms) AS MaximumWaitMilliseconds - FROM sys.query_store_wait_stats AS ws - INNER JOIN sys.query_store_runtime_stats_interval AS rsi - ON rsi.runtime_stats_interval_id = ws.runtime_stats_interval_id - WHERE @WaitStatsStatus = 'Available' - AND ws.execution_type = 0 - AND rsi.start_time < @ResolvedEndTime - AND rsi.end_time > @ResolvedStartTime - ), - WaitCategories AS - ( - SELECT - ws.PlanId, - ws.WaitCategoryDescription, - CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds, - MAX(ws.MaximumWaitMilliseconds) AS MaximumWaitMilliseconds - FROM WaitStatsRows AS ws - GROUP BY - ws.PlanId, - ws.WaitCategoryDescription - ), - AggregatedWaitStats AS - ( - SELECT - ws.PlanId, - CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds - FROM WaitCategories AS ws - GROUP BY ws.PlanId - ), - WaitStatsXml AS - ( - SELECT - wp.PlanId, - ( - SELECT - wc.WaitCategoryDescription AS [@Category], - wc.TotalWaitMilliseconds AS [@TotalWaitMilliseconds], - CONVERT(decimal(38, 4), wc.TotalWaitMilliseconds / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) AS [@AverageWaitMilliseconds], - wc.MaximumWaitMilliseconds AS [@MaximumWaitMilliseconds] - FROM WaitCategories AS wc - WHERE wc.PlanId = wp.PlanId - AND wc.TotalWaitMilliseconds > 0 - ORDER BY - wc.TotalWaitMilliseconds DESC, - wc.WaitCategoryDescription ASC - FOR XML PATH(N'WaitCategory'), ROOT(N'WaitStats'), TYPE - ) AS WaitStatsXml - FROM (SELECT DISTINCT PlanId FROM WaitCategories) AS wp - INNER JOIN AggregatedRuntimeStats AS ars - ON ars.PlanId = wp.PlanId ) SELECT q.query_id AS QueryId, @@ -343,17 +275,7 @@ BEGIN p.compatibility_level AS CompatibilityLevel, p.is_online_index_plan AS IsOnlineIndexPlan, p.is_trivial_plan AS IsTrivialPlan, - p.is_parallel_plan AS IsParallelPlan, - CASE - WHEN @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) - END AS TotalWaitMilliseconds, - CASE - WHEN @WaitStatsStatus = 'Available' THEN CONVERT(decimal(38, 4), ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) - END AS AverageWaitMilliseconds, - @WaitStatsStatus AS WaitStatsStatus, - CASE - WHEN @WaitStatsStatus = 'Available' THEN ISNULL(wsx.WaitStatsXml, CONVERT(xml, N'')) - END AS WaitStatsXml + p.is_parallel_plan AS IsParallelPlan FROM AggregatedRuntimeStats AS ars INNER JOIN sys.query_store_plan AS p ON p.plan_id = ars.PlanId @@ -361,16 +283,11 @@ BEGIN ON q.query_id = p.query_id INNER JOIN sys.query_store_query_text AS qt ON qt.query_text_id = q.query_text_id - LEFT JOIN AggregatedWaitStats AS aws - ON aws.PlanId = ars.PlanId - LEFT JOIN WaitStatsXml AS wsx - ON wsx.PlanId = ars.PlanId WHERE ISNULL(q.object_id, -1) <> ISNULL(@SlowQueriesProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@PlanDiagnosticsProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@StatisticsHealthProcedureObjectId, -2) AND (@QueryTextContains IS NULL OR qt.query_sql_text LIKE @QueryTextPattern ESCAPE N'~') ORDER BY - CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' AND aws.TotalWaitMilliseconds IS NULL THEN 1 ELSE 0 END ASC, CASE WHEN @OrderByNormalized = 'TotalDuration' THEN ars.TotalDurationMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'AverageDuration' THEN ars.AverageDurationMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'MaximumDuration' THEN ars.MaximumDurationMicroseconds END DESC, @@ -378,7 +295,6 @@ BEGIN CASE WHEN @OrderByNormalized = 'AverageCpu' THEN ars.AverageCpuMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'LogicalReads' THEN ars.TotalLogicalReads END DESC, CASE WHEN @OrderByNormalized = 'Executions' THEN ars.RegularExecutionCount END DESC, - CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) END DESC, q.query_id ASC, p.plan_id ASC OFFSET @Offset ROWS FETCH NEXT @Top ROWS ONLY; @@ -422,7 +338,6 @@ BEGIN ,@Start datetime = GETUTCDATE() ,@Rows int = 0 ,@QueryStoreState nvarchar(60) - ,@QueryStoreReadonlyReason bigint ,@AuditText nvarchar(3500) ,@FoundPlanId bigint ,@QueryId bigint @@ -452,8 +367,6 @@ BEGIN ,@LocalPlanXml xml ,@SanitizedShowPlanXml xml ,@SerializedPlanXml nvarchar(max) - ,@ParameterListCount bigint - ,@ParameterListRemoved bigint = 0 ,@RemainingParameterListCount bigint ,@ForbiddenAttributeCount bigint ,@SanitizationStatus varchar(32) @@ -474,16 +387,8 @@ BEGIN THROW 50001, 'Plan ID must be a positive bigint.', 1 SELECT @QueryStoreState = actual_state_desc - ,@QueryStoreReadonlyReason = readonly_reason FROM sys.database_query_store_options - SET @AuditText = CONCAT( - @AuditText, - N';QueryStoreState=', ISNULL(CONVERT(nvarchar(60), @QueryStoreState), N'Unknown'), - N';QueryStoreReadonlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadonlyReason), N'NULL')) - - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Run',@Text=@AuditText - IF @QueryStoreState IS NULL OR @QueryStoreState NOT IN ('READ_WRITE', 'READ_ONLY') THROW 50002, 'Query Store is not readable.', 1 @@ -543,13 +448,7 @@ BEGIN BEGIN BEGIN TRY SET @SanitizedShowPlanXml = @LocalPlanXml - SET @ParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') - - WHILE @ParameterListRemoved < @ParameterListCount - BEGIN - SET @SanitizedShowPlanXml.modify('delete (//*[local-name(.) = "ParameterList"])[1]') - SET @ParameterListRemoved = @ParameterListRemoved + 1 - END + SET @SanitizedShowPlanXml.modify('delete //*[local-name(.) = "ParameterList"]') SET @RemainingParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') SET @ForbiddenAttributeCount = @SanitizedShowPlanXml.value('count(//@*[local-name(.) = "ParameterCompiledValue" or local-name(.) = "ParameterRuntimeValue"])', 'bigint') @@ -670,25 +569,25 @@ BEGIN IF @Top IS NULL OR @Top < 1 OR @Top > 100 BEGIN - RAISERROR('@Top must be between 1 and 100.', 18, 127); + THROW 50000, '@Top must be between 1 and 100.', 127; END IF @Offset IS NULL OR @Offset < 0 OR @Offset > 10000 BEGIN - RAISERROR('@Offset must be between 0 and 10000.', 18, 127); + THROW 50000, '@Offset must be between 0 and 10000.', 127; END IF @NormalizedOrderBy IS NULL OR @NormalizedOrderBy NOT IN ('MODIFICATIONCOUNT', 'MODIFICATIONPERCENT', 'LASTUPDATED', 'SAMPLINGPERCENT', 'ROWS') BEGIN - RAISERROR('@OrderBy must be ModificationCount, ModificationPercent, LastUpdated, SamplingPercent, or Rows.', 18, 127); + THROW 50000, '@OrderBy must be ModificationCount, ModificationPercent, LastUpdated, SamplingPercent, or Rows.', 127; END IF @TableName IS NOT NULL BEGIN IF LEN(LTRIM(RTRIM(@TableName))) = 0 BEGIN - RAISERROR('@TableName must be nonblank when supplied.', 18, 127); + THROW 50000, '@TableName must be nonblank when supplied.', 127; END SELECT @@ -700,12 +599,12 @@ BEGIN IF @TableCount = 0 BEGIN - RAISERROR('@TableName does not resolve to a user table.', 18, 127); + THROW 50000, '@TableName does not resolve to a user table.', 127; END IF @TableCount > 1 BEGIN - RAISERROR('@TableName must resolve to exactly one user table.', 18, 127); + THROW 50000, '@TableName must resolve to exactly one user table.', 127; END END diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql index 5a6ae50eec..a1e2016db3 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql @@ -11,7 +11,6 @@ BEGIN ,@Start datetime = GETUTCDATE() ,@Rows int = 0 ,@QueryStoreState nvarchar(60) - ,@QueryStoreReadonlyReason bigint ,@AuditText nvarchar(3500) ,@FoundPlanId bigint ,@QueryId bigint @@ -41,8 +40,6 @@ BEGIN ,@LocalPlanXml xml ,@SanitizedShowPlanXml xml ,@SerializedPlanXml nvarchar(max) - ,@ParameterListCount bigint - ,@ParameterListRemoved bigint = 0 ,@RemainingParameterListCount bigint ,@ForbiddenAttributeCount bigint ,@SanitizationStatus varchar(32) @@ -63,16 +60,8 @@ BEGIN THROW 50001, 'Plan ID must be a positive bigint.', 1 SELECT @QueryStoreState = actual_state_desc - ,@QueryStoreReadonlyReason = readonly_reason FROM sys.database_query_store_options - SET @AuditText = CONCAT( - @AuditText, - N';QueryStoreState=', ISNULL(CONVERT(nvarchar(60), @QueryStoreState), N'Unknown'), - N';QueryStoreReadonlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadonlyReason), N'NULL')) - - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Run',@Text=@AuditText - IF @QueryStoreState IS NULL OR @QueryStoreState NOT IN ('READ_WRITE', 'READ_ONLY') THROW 50002, 'Query Store is not readable.', 1 @@ -132,13 +121,7 @@ BEGIN BEGIN BEGIN TRY SET @SanitizedShowPlanXml = @LocalPlanXml - SET @ParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') - - WHILE @ParameterListRemoved < @ParameterListCount - BEGIN - SET @SanitizedShowPlanXml.modify('delete (//*[local-name(.) = "ParameterList"])[1]') - SET @ParameterListRemoved = @ParameterListRemoved + 1 - END + SET @SanitizedShowPlanXml.modify('delete //*[local-name(.) = "ParameterList"]') SET @RemainingParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') SET @ForbiddenAttributeCount = @SanitizedShowPlanXml.value('count(//@*[local-name(.) = "ParameterCompiledValue" or local-name(.) = "ParameterRuntimeValue"])', 'bigint') diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql index 08c583d6c6..812c30fcf4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql @@ -25,8 +25,6 @@ BEGIN DECLARE @QueryTextFilterLength int; DECLARE @QueryStoreState nvarchar(60); DECLARE @QueryStoreReadOnlyReason bigint; - DECLARE @WaitStatsCaptureMode nvarchar(60); - DECLARE @WaitStatsStatus varchar(32); DECLARE @SlowQueriesProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStoreSlowQueries'); DECLARE @PlanDiagnosticsProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStorePlanDiagnostics'); DECLARE @StatisticsHealthProcedureObjectId int = OBJECT_ID(N'dbo.GetStatisticsHealth'); @@ -50,16 +48,9 @@ BEGIN SELECT @QueryStoreState = actual_state_desc, - @QueryStoreReadOnlyReason = readonly_reason, - @WaitStatsCaptureMode = wait_stats_capture_mode_desc + @QueryStoreReadOnlyReason = readonly_reason FROM sys.database_query_store_options; - SET @WaitStatsStatus = - CASE @WaitStatsCaptureMode - WHEN N'ON' THEN 'Available' - WHEN N'OFF' THEN 'Disabled' - ELSE 'Unavailable' - END; SET @AuditText = CONCAT( @AuditText, N';StartTimeUtc=', CONVERT(nvarchar(33), @ResolvedStartTime, 127), @@ -71,8 +62,7 @@ BEGIN N';QueryTextFilterPresent=', CASE WHEN @QueryTextContains IS NULL THEN N'0' ELSE N'1' END, N';QueryTextFilterLength=', @QueryTextFilterLength, N';QueryStoreState=', ISNULL(@QueryStoreState, N'Unknown'), - N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown'), - N';WaitStatsCaptureMode=', ISNULL(@WaitStatsCaptureMode, N'Unknown')); + N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown')); EXECUTE dbo.LogEvent @Process = @ProcedureName, @@ -112,7 +102,6 @@ BEGIN WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageCpu' THEN 'AverageCpu' WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'LogicalReads' THEN 'LogicalReads' WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'Executions' THEN 'Executions' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalWait' THEN 'TotalWait' END; IF @OrderByNormalized IS NULL @@ -228,63 +217,6 @@ BEGIN FROM RankedCollapsedRuntimeStats AS rs GROUP BY rs.PlanId HAVING SUM(rs.RegularExecutionCount) >= @MinExecutions - ), - WaitStatsRows AS - ( - SELECT - ws.plan_id AS PlanId, - ws.wait_category_desc AS WaitCategoryDescription, - CONVERT(decimal(38, 0), ws.total_query_wait_time_ms) AS TotalWaitMilliseconds, - CONVERT(decimal(38, 0), ws.max_query_wait_time_ms) AS MaximumWaitMilliseconds - FROM sys.query_store_wait_stats AS ws - INNER JOIN sys.query_store_runtime_stats_interval AS rsi - ON rsi.runtime_stats_interval_id = ws.runtime_stats_interval_id - WHERE @WaitStatsStatus = 'Available' - AND ws.execution_type = 0 - AND rsi.start_time < @ResolvedEndTime - AND rsi.end_time > @ResolvedStartTime - ), - WaitCategories AS - ( - SELECT - ws.PlanId, - ws.WaitCategoryDescription, - CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds, - MAX(ws.MaximumWaitMilliseconds) AS MaximumWaitMilliseconds - FROM WaitStatsRows AS ws - GROUP BY - ws.PlanId, - ws.WaitCategoryDescription - ), - AggregatedWaitStats AS - ( - SELECT - ws.PlanId, - CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds - FROM WaitCategories AS ws - GROUP BY ws.PlanId - ), - WaitStatsXml AS - ( - SELECT - wp.PlanId, - ( - SELECT - wc.WaitCategoryDescription AS [@Category], - wc.TotalWaitMilliseconds AS [@TotalWaitMilliseconds], - CONVERT(decimal(38, 4), wc.TotalWaitMilliseconds / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) AS [@AverageWaitMilliseconds], - wc.MaximumWaitMilliseconds AS [@MaximumWaitMilliseconds] - FROM WaitCategories AS wc - WHERE wc.PlanId = wp.PlanId - AND wc.TotalWaitMilliseconds > 0 - ORDER BY - wc.TotalWaitMilliseconds DESC, - wc.WaitCategoryDescription ASC - FOR XML PATH(N'WaitCategory'), ROOT(N'WaitStats'), TYPE - ) AS WaitStatsXml - FROM (SELECT DISTINCT PlanId FROM WaitCategories) AS wp - INNER JOIN AggregatedRuntimeStats AS ars - ON ars.PlanId = wp.PlanId ) SELECT q.query_id AS QueryId, @@ -323,17 +255,7 @@ BEGIN p.compatibility_level AS CompatibilityLevel, p.is_online_index_plan AS IsOnlineIndexPlan, p.is_trivial_plan AS IsTrivialPlan, - p.is_parallel_plan AS IsParallelPlan, - CASE - WHEN @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) - END AS TotalWaitMilliseconds, - CASE - WHEN @WaitStatsStatus = 'Available' THEN CONVERT(decimal(38, 4), ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) - END AS AverageWaitMilliseconds, - @WaitStatsStatus AS WaitStatsStatus, - CASE - WHEN @WaitStatsStatus = 'Available' THEN ISNULL(wsx.WaitStatsXml, CONVERT(xml, N'')) - END AS WaitStatsXml + p.is_parallel_plan AS IsParallelPlan FROM AggregatedRuntimeStats AS ars INNER JOIN sys.query_store_plan AS p ON p.plan_id = ars.PlanId @@ -341,16 +263,11 @@ BEGIN ON q.query_id = p.query_id INNER JOIN sys.query_store_query_text AS qt ON qt.query_text_id = q.query_text_id - LEFT JOIN AggregatedWaitStats AS aws - ON aws.PlanId = ars.PlanId - LEFT JOIN WaitStatsXml AS wsx - ON wsx.PlanId = ars.PlanId WHERE ISNULL(q.object_id, -1) <> ISNULL(@SlowQueriesProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@PlanDiagnosticsProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@StatisticsHealthProcedureObjectId, -2) AND (@QueryTextContains IS NULL OR qt.query_sql_text LIKE @QueryTextPattern ESCAPE N'~') ORDER BY - CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' AND aws.TotalWaitMilliseconds IS NULL THEN 1 ELSE 0 END ASC, CASE WHEN @OrderByNormalized = 'TotalDuration' THEN ars.TotalDurationMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'AverageDuration' THEN ars.AverageDurationMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'MaximumDuration' THEN ars.MaximumDurationMicroseconds END DESC, @@ -358,7 +275,6 @@ BEGIN CASE WHEN @OrderByNormalized = 'AverageCpu' THEN ars.AverageCpuMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'LogicalReads' THEN ars.TotalLogicalReads END DESC, CASE WHEN @OrderByNormalized = 'Executions' THEN ars.RegularExecutionCount END DESC, - CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) END DESC, q.query_id ASC, p.plan_id ASC OFFSET @Offset ROWS FETCH NEXT @Top ROWS ONLY; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql index d6b18554a9..2df69dff55 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql @@ -29,25 +29,25 @@ BEGIN IF @Top IS NULL OR @Top < 1 OR @Top > 100 BEGIN - RAISERROR('@Top must be between 1 and 100.', 18, 127); + THROW 50000, '@Top must be between 1 and 100.', 127; END IF @Offset IS NULL OR @Offset < 0 OR @Offset > 10000 BEGIN - RAISERROR('@Offset must be between 0 and 10000.', 18, 127); + THROW 50000, '@Offset must be between 0 and 10000.', 127; END IF @NormalizedOrderBy IS NULL OR @NormalizedOrderBy NOT IN ('MODIFICATIONCOUNT', 'MODIFICATIONPERCENT', 'LASTUPDATED', 'SAMPLINGPERCENT', 'ROWS') BEGIN - RAISERROR('@OrderBy must be ModificationCount, ModificationPercent, LastUpdated, SamplingPercent, or Rows.', 18, 127); + THROW 50000, '@OrderBy must be ModificationCount, ModificationPercent, LastUpdated, SamplingPercent, or Rows.', 127; END IF @TableName IS NOT NULL BEGIN IF LEN(LTRIM(RTRIM(@TableName))) = 0 BEGIN - RAISERROR('@TableName must be nonblank when supplied.', 18, 127); + THROW 50000, '@TableName must be nonblank when supplied.', 127; END SELECT @@ -59,12 +59,12 @@ BEGIN IF @TableCount = 0 BEGIN - RAISERROR('@TableName does not resolve to a user table.', 18, 127); + THROW 50000, '@TableName does not resolve to a user table.', 127; END IF @TableCount > 1 BEGIN - RAISERROR('@TableName must resolve to exactly one user table.', 18, 127); + THROW 50000, '@TableName must resolve to exactly one user table.', 127; END END diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs index 36067d83eb..3ed5f639fc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs @@ -74,6 +74,7 @@ public async Task GivenAQueryStoreCapturedFhirQuery_WhenDiagnosticsProceduresAre await AssertPlanDiagnosticsAsync(connection, planId, queryMarker, CancellationToken.None); await AssertResourceStatisticsHealthAsync(connection, CancellationToken.None); + await AssertTotalWaitOrderingIsRejectedAsync(connection, CancellationToken.None); } private static async Task EnableAndVerifyQueryStoreAsync(SqlConnection connection, CancellationToken cancellationToken) @@ -160,12 +161,33 @@ private static async Task GetSlowQueryPlanIdAsync( Assert.Contains(queryMarker, reader.GetString(reader.GetOrdinal("QuerySqlText"))); Assert.False(reader.IsDBNull(reader.GetOrdinal("FirstExecutionTimeUtc"))); Assert.False(reader.IsDBNull(reader.GetOrdinal("LastExecutionTimeUtc"))); + AssertWaitColumnsAreNotReturned(reader); Assert.False(await reader.ReadAsync(cancellationToken), "The unique query-text filter returned more than one Query Store plan."); Assert.False(await reader.NextResultAsync(cancellationToken), "dbo.GetQueryStoreSlowQueries returned more than one result set."); return planId; } + private static void AssertWaitColumnsAreNotReturned(SqlDataReader reader) + { + Assert.Throws(() => reader.GetOrdinal("TotalWaitMilliseconds")); + Assert.Throws(() => reader.GetOrdinal("AverageWaitMilliseconds")); + Assert.Throws(() => reader.GetOrdinal("WaitStatsStatus")); + Assert.Throws(() => reader.GetOrdinal("WaitStatsXml")); + } + + private static async Task AssertTotalWaitOrderingIsRejectedAsync(SqlConnection connection, CancellationToken cancellationToken) + { + using SqlCommand command = connection.CreateCommand(); + command.CommandType = CommandType.StoredProcedure; + command.CommandText = "dbo.GetQueryStoreSlowQueries"; + command.Parameters.Add("@OrderBy", SqlDbType.VarChar, 32).Value = "TotalWait"; + + SqlException exception = await Assert.ThrowsAsync(() => command.ExecuteNonQueryAsync(cancellationToken)); + + Assert.Contains("@OrderBy is not supported.", exception.Message); + } + private static async Task AssertPlanDiagnosticsAsync( SqlConnection connection, long planId, From 3b32365b6981c4fe3f45a01b0ce616ec7a585bdd Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 14 Aug 2026 01:40:34 +0000 Subject: [PATCH 07/20] Restore self-contained wait diagnostics Return Query Store waits with slow-query results so direct SQL and future Geneva callers can retrieve the complete diagnostic payload without joining Log Analytics.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 62 +++++++++---- .../Features/Schema/Migrations/117.diff.sql | 90 ++++++++++++++++++- .../Sql/Sprocs/GetQueryStoreSlowQueries.sql | 90 ++++++++++++++++++- .../SqlServerQueryStoreDiagnosticsTests.cs | 39 ++++---- 4 files changed, 238 insertions(+), 43 deletions(-) diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 5050b32096..ba4353c9bc 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -6,11 +6,11 @@ Agreed baseline for implementation. This document defines the SQL contract, secu ## Problem -FHIR Azure SQL performance investigations currently require privileged, manual access to Query Store plans, runtime metrics, and statistics metadata. Query Store wait statistics are supplied separately through the Azure SQL `QueryStoreWaitStatistics` diagnostic setting in Log Analytics. That diagnostic stream does not provide query text or Showplan XML. Support engineers still need a bounded way to: +FHIR Azure SQL performance investigations currently require privileged, manual access to Query Store plans, runtime metrics, wait statistics, and statistics metadata. Although Azure SQL can export `QueryStoreWaitStatistics` to Log Analytics, the SQL diagnostics intentionally include plan-level waits so direct SQL and future Geneva callers receive one self-contained slow-query result with runtime metrics, waits, query text, and plan IDs without joining Log Analytics. Support engineers still need a bounded way to: - identify expensive or regressed query plans; - retrieve an SSMS-viewable Query Store Showplan; -- inspect runtime metrics; and +- compare runtime and wait metrics; and - inspect statistics freshness, sampling, and cardinality metadata. The baseline is a self-contained, read-only SQL interface. Filtering, validation, redaction, paging, permissions, and auditing must live in SQL so the procedures can be used by an authorized direct SQL connection or wrapped by future operational tooling. @@ -29,15 +29,15 @@ Related internal guidance: 1. Identify slow or resource-intensive query plans over a bounded time range. 2. Return full Query Store query text to authorized diagnostic callers. 3. Return an SSMS-viewable Query Store Showplan after removing parameter-value metadata. -4. Report statistics freshness, sampling, and filter metadata without returning histogram values. -5. Provide an execute-only database role for least-privilege callers. -6. Keep the SQL contract independent of any API, Geneva action, or other caller implementation. +4. Include Query Store wait statistics in slow-query results when capture is available. +5. Report statistics freshness, sampling, and filter metadata without returning histogram values. +6. Provide an execute-only database role for least-privilege callers. +7. Keep the SQL contract independent of any API, Geneva action, or other caller implementation. ## Non-goals - Retrieving or capturing an actual execution plan. - Reconstructing or executing SQL from Query Store. -- Returning plan-level wait statistics or duplicating the Azure SQL `QueryStoreWaitStatistics` Log Analytics stream. - Returning statistics histograms, density vectors, or sampled column values. - Clearing the procedure cache, updating statistics, forcing plans, or changing Query Store configuration. - Providing caller concurrency control, circuit breaking, command timeouts, artifact retention, or download policy. @@ -52,9 +52,9 @@ The baseline targets Azure SQL Database and uses only Query Store catalog column ### Query Store wait-stat observability -Query Store wait statistics come from the Azure SQL `QueryStoreWaitStatistics` diagnostic setting in Log Analytics and are intentionally not duplicated by this SQL interface. Log Analytics does not supply the query text or Showplan XML returned by the diagnostic procedures. +When `sys.database_query_store_options.wait_stats_capture_mode_desc` is `ON`, the slow-query procedure reads `sys.query_store_wait_stats` directly. `WaitStatsStatus` is `Available` for `ON`, `Disabled` for `OFF`, and `Unavailable` for any other value. This lets authorized direct SQL and future Geneva callers retrieve the complete slow-query diagnostic payload—runtime metrics, waits, query text, and plan IDs—from one interface. Azure SQL may also export `QueryStoreWaitStatistics` to Log Analytics, but that stream does not provide the query text or Showplan XML returned by these procedures. -`DatabaseWaitStatistics`, when enabled, is a separate database-level diagnostic stream. It is not plan-level data and is not a substitute for `QueryStoreWaitStatistics`. +`DatabaseWaitStatistics`, when enabled, remains separate database-level telemetry. It is not plan-level data and is not a substitute for the Query Store wait statistics returned with each slow-query plan. ### Query Store plans are estimated plans @@ -71,7 +71,6 @@ This is documentation only. No stub procedure, shared output contract, permissio ## Implementation simplifications - Plan-type and Parameter Sensitive Plan dispatcher/query-variant metadata are intentionally not read. Those Azure SQL catalog fields are not stable across the supported deployment fleet, so both Query Store procedures omit them rather than returning speculative NULL/status fields or attempting version-specific fallback logic. -- `GetQueryStoreSlowQueries` aggregates runtime data only. Wait analysis remains in the Azure SQL `QueryStoreWaitStatistics` Log Analytics stream; the procedure neither queries nor returns plan-level wait data. `DatabaseWaitStatistics` remains separate database-level telemetry. - `GetStatisticsHealth` reports database-level `sys.dm_db_stats_properties` metadata for each statistics object. It does not expand incremental statistics into partition-level property rows; unavailable properties remain `NULL` and are explicitly marked `PropertiesUnavailable`. Its table-name input is materialized as `nvarchar(128)`, rather than the type-equivalent `sysname` alias, because the existing schema C# model generator interprets `sysname` as a table-valued parameter. ### Accepted query and plan content @@ -125,8 +124,6 @@ The rollout order is: 4. provision approved role membership; and 5. enable the PaaS invocation and artifact-handling workflow. -Provisioning and retaining the Azure SQL `QueryStoreWaitStatistics` diagnostic setting and its Log Analytics destination is a separate observability rollout. It is not a schema prerequisite or a source for query text or Showplan XML. - ## Stored procedures All procedures: @@ -177,10 +174,11 @@ The `@OrderBy` allowlist is: - `AverageCpu` - `LogicalReads` - `Executions` +- `TotalWait` -Unknown order values fail explicitly. Ordering uses a static `CASE` expression rather than dynamic SQL. Diagnostic metrics sort descending, and `query_id ASC, plan_id ASC` are deterministic tie-breakers. +Unknown order values fail explicitly. Ordering uses a static `CASE` expression rather than dynamic SQL. Diagnostic metrics sort descending, NULL wait totals sort last, and `query_id ASC, plan_id ASC` are deterministic tie-breakers. -There is no execution-type input in the baseline. Runtime metrics include regular executions only. The implementation should contain a focused comment identifying where execution-type support could be added later. +There is no execution-type input in the baseline. Runtime and wait metrics include regular executions only. The implementation should contain a focused comment identifying where execution-type support could be added later. #### Time-window semantics @@ -211,6 +209,28 @@ Query-level compile count and last compile time are repeated on each plan row an Plans with fewer than `@MinExecutions` regular executions in the selected window are excluded. The diagnostic procedures' own Query Store entries are also excluded. All other object-bound and ad hoc Query Store entries are eligible. +#### Wait statistics + +Wait statistics are aggregated for the same regular-execution population and overlapping intervals as runtime metrics. + +Each result row contains: + +- `TotalWaitMilliseconds` +- `AverageWaitMilliseconds` +- `WaitStatsStatus` +- `WaitStatsXml` + +`WaitStatsXml` contains one element per wait category, ordered by total wait descending, with: + +- category name; +- total wait milliseconds; +- average wait milliseconds; and +- maximum wait milliseconds. + +Zero-wait categories are omitted. When wait capture is available but a plan has no waits, the value is an empty typed root such as ``. When wait capture is disabled or unavailable, `WaitStatsXml` and scalar wait metrics are NULL and `WaitStatsStatus` explains the condition. Other runtime results still return. + +If `@OrderBy = 'TotalWait'` while wait capture is disabled or unavailable, rows still return. NULL wait totals sort last. + #### Output The single result set includes: @@ -233,6 +253,9 @@ The single result set includes: - query-level compile count and last compile time - forced-plan state and available force-failure metadata - other universally available diagnostic plan metadata; plan-type, dispatcher, and query-variant metadata are omitted +- total and average wait milliseconds +- `WaitStatsStatus` +- `WaitStatsXml` Physical reads and writes are output metrics but are not ordering options. Query context/handle metadata and execution type are omitted. @@ -416,7 +439,7 @@ SQL enforces: - a 3-256-character literal query-text substring; - one plan per plan-diagnostics call; - static SQL only; -- regular-execution-only runtime aggregation; and +- regular-execution-only runtime and wait aggregation; and - exclusion of the diagnostic procedures' own Query Store entries. Limits are hard-coded in the procedures. There is no `dbo.Parameters` kill switch, SQL concurrency gate, plan-size cap, total-count query, continuation token, or `HasMoreRows` result. @@ -435,7 +458,7 @@ Audit records include: - returned row count; and - sanitized XML size for successful plan retrieval. -Slow-query audit metadata includes resolved UTC window, ordering mode, `@Top`, `@Offset`, `@MinExecutions`, and query-text filter presence/length, but never the filter text. +Slow-query audit metadata includes resolved UTC window, ordering mode, `@Top`, `@Offset`, `@MinExecutions`, Query Store state/read-only reason, wait-statistics capture mode, and query-text filter presence/length, but never the filter text or wait payload. Plan audit metadata includes `plan_id`, sanitization status, stable error code, and result size, but never query text or XML. @@ -454,20 +477,21 @@ If Start, End, or Error logging fails, the diagnostic call fails. Callers do not ## Testing requirements -**Deferred prototype validation:** The prototype has one representative SQL-backed end-to-end path. Exhaustive matrix validation remains future work, including negative validation, fail-closed audit behavior, permissions, and a malformed-fixture corpus. +**Deferred prototype validation:** The prototype has one representative SQL-backed end-to-end path. Exhaustive matrix validation remains future work, including negative validation, fail-closed audit behavior, permissions, a malformed-fixture corpus, and wait-disabled cases. ### Slow-query aggregation 1. Duplicate active-interval in-memory/persisted rows are collapsed before rollup. 2. Weighted totals and averages use the agreed decimal precision. 3. Minimum, maximum, and deterministic last-value calculations are correct. -4. Only regular executions contribute to runtime metrics. +4. Only regular executions contribute to runtime and wait metrics. 5. Overlapping Query Store interval semantics are verified at both window boundaries. 6. Time, row, offset, minimum-execution, query-text, and order allowlists cannot be bypassed. 7. Literal query-text matching correctly escapes `~`, `%`, `_`, and `[`. 8. Query Store `READ_WRITE` and readable `READ_ONLY` states return data. 9. Query Store `OFF`, `ERROR`, and unreadable states fail with actionable errors. -10. Diagnostic procedures exclude their own Query Store entries. +10. Wait capture available, disabled, unavailable, empty, and `TotalWait` ordering cases are covered. +11. Diagnostic procedures exclude their own Query Store entries. ### Showplan sanitization @@ -517,7 +541,6 @@ If Start, End, or Error logging fails, the diagnostic call fails. Callers do not - Update the OSS FHIR package versions and synchronized target schema version through the existing `fhir-paas` dependency flow. - Do not copy the stored procedure or role DDL into PaaS Script Runner scripts. - Add role membership only for the approved operational identity. -- Configure and operate `QueryStoreWaitStatistics` through the Azure SQL diagnostic setting and Log Analytics independently of stored-procedure deployment. - Implement the selected caller, result transport, artifact storage, and operational authorization in `fhir-paas`. - Deploy schema/package consumption before enabling the caller. - Control caller rollout and role membership independently in each environment. @@ -527,6 +550,7 @@ If Start, End, or Error logging fails, the diagnostic call fails. Callers do not - [Monitor performance by using Query Store](https://learn.microsoft.com/sql/relational-databases/performance/monitoring-performance-by-using-the-query-store) - [How Query Store collects data](https://learn.microsoft.com/sql/relational-databases/performance/how-query-store-collects-data) - [`sys.query_store_runtime_stats`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-runtime-stats-transact-sql) +- [`sys.query_store_wait_stats`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-wait-stats-transact-sql) - [`sys.query_store_plan`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-plan-transact-sql) - [`sys.query_store_query`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-query-transact-sql) - [`sys.query_store_query_text`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-query-text-transact-sql) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql index 8109bc4936..8f51714398 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql @@ -45,6 +45,8 @@ BEGIN DECLARE @QueryTextFilterLength int; DECLARE @QueryStoreState nvarchar(60); DECLARE @QueryStoreReadOnlyReason bigint; + DECLARE @WaitStatsCaptureMode nvarchar(60); + DECLARE @WaitStatsStatus varchar(32); DECLARE @SlowQueriesProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStoreSlowQueries'); DECLARE @PlanDiagnosticsProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStorePlanDiagnostics'); DECLARE @StatisticsHealthProcedureObjectId int = OBJECT_ID(N'dbo.GetStatisticsHealth'); @@ -68,9 +70,16 @@ BEGIN SELECT @QueryStoreState = actual_state_desc, - @QueryStoreReadOnlyReason = readonly_reason + @QueryStoreReadOnlyReason = readonly_reason, + @WaitStatsCaptureMode = wait_stats_capture_mode_desc FROM sys.database_query_store_options; + SET @WaitStatsStatus = + CASE @WaitStatsCaptureMode + WHEN N'ON' THEN 'Available' + WHEN N'OFF' THEN 'Disabled' + ELSE 'Unavailable' + END; SET @AuditText = CONCAT( @AuditText, N';StartTimeUtc=', CONVERT(nvarchar(33), @ResolvedStartTime, 127), @@ -82,7 +91,8 @@ BEGIN N';QueryTextFilterPresent=', CASE WHEN @QueryTextContains IS NULL THEN N'0' ELSE N'1' END, N';QueryTextFilterLength=', @QueryTextFilterLength, N';QueryStoreState=', ISNULL(@QueryStoreState, N'Unknown'), - N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown')); + N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown'), + N';WaitStatsCaptureMode=', ISNULL(@WaitStatsCaptureMode, N'Unknown')); EXECUTE dbo.LogEvent @Process = @ProcedureName, @@ -122,6 +132,7 @@ BEGIN WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageCpu' THEN 'AverageCpu' WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'LogicalReads' THEN 'LogicalReads' WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'Executions' THEN 'Executions' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalWait' THEN 'TotalWait' END; IF @OrderByNormalized IS NULL @@ -237,6 +248,63 @@ BEGIN FROM RankedCollapsedRuntimeStats AS rs GROUP BY rs.PlanId HAVING SUM(rs.RegularExecutionCount) >= @MinExecutions + ), + WaitStatsRows AS + ( + SELECT + ws.plan_id AS PlanId, + ws.wait_category_desc AS WaitCategoryDescription, + CONVERT(decimal(38, 0), ws.total_query_wait_time_ms) AS TotalWaitMilliseconds, + CONVERT(decimal(38, 0), ws.max_query_wait_time_ms) AS MaximumWaitMilliseconds + FROM sys.query_store_wait_stats AS ws + INNER JOIN sys.query_store_runtime_stats_interval AS rsi + ON rsi.runtime_stats_interval_id = ws.runtime_stats_interval_id + WHERE @WaitStatsStatus = 'Available' + AND ws.execution_type = 0 + AND rsi.start_time < @ResolvedEndTime + AND rsi.end_time > @ResolvedStartTime + ), + WaitCategories AS + ( + SELECT + ws.PlanId, + ws.WaitCategoryDescription, + CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds, + MAX(ws.MaximumWaitMilliseconds) AS MaximumWaitMilliseconds + FROM WaitStatsRows AS ws + GROUP BY + ws.PlanId, + ws.WaitCategoryDescription + ), + AggregatedWaitStats AS + ( + SELECT + ws.PlanId, + CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds + FROM WaitCategories AS ws + GROUP BY ws.PlanId + ), + WaitStatsXml AS + ( + SELECT + wp.PlanId, + ( + SELECT + wc.WaitCategoryDescription AS [@Category], + wc.TotalWaitMilliseconds AS [@TotalWaitMilliseconds], + CONVERT(decimal(38, 4), wc.TotalWaitMilliseconds / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) AS [@AverageWaitMilliseconds], + wc.MaximumWaitMilliseconds AS [@MaximumWaitMilliseconds] + FROM WaitCategories AS wc + WHERE wc.PlanId = wp.PlanId + AND wc.TotalWaitMilliseconds > 0 + ORDER BY + wc.TotalWaitMilliseconds DESC, + wc.WaitCategoryDescription ASC + FOR XML PATH(N'WaitCategory'), ROOT(N'WaitStats'), TYPE + ) AS WaitStatsXml + FROM (SELECT DISTINCT PlanId FROM WaitCategories) AS wp + INNER JOIN AggregatedRuntimeStats AS ars + ON ars.PlanId = wp.PlanId ) SELECT q.query_id AS QueryId, @@ -275,7 +343,17 @@ BEGIN p.compatibility_level AS CompatibilityLevel, p.is_online_index_plan AS IsOnlineIndexPlan, p.is_trivial_plan AS IsTrivialPlan, - p.is_parallel_plan AS IsParallelPlan + p.is_parallel_plan AS IsParallelPlan, + CASE + WHEN @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) + END AS TotalWaitMilliseconds, + CASE + WHEN @WaitStatsStatus = 'Available' THEN CONVERT(decimal(38, 4), ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) + END AS AverageWaitMilliseconds, + @WaitStatsStatus AS WaitStatsStatus, + CASE + WHEN @WaitStatsStatus = 'Available' THEN ISNULL(wsx.WaitStatsXml, CONVERT(xml, N'')) + END AS WaitStatsXml FROM AggregatedRuntimeStats AS ars INNER JOIN sys.query_store_plan AS p ON p.plan_id = ars.PlanId @@ -283,11 +361,16 @@ BEGIN ON q.query_id = p.query_id INNER JOIN sys.query_store_query_text AS qt ON qt.query_text_id = q.query_text_id + LEFT JOIN AggregatedWaitStats AS aws + ON aws.PlanId = ars.PlanId + LEFT JOIN WaitStatsXml AS wsx + ON wsx.PlanId = ars.PlanId WHERE ISNULL(q.object_id, -1) <> ISNULL(@SlowQueriesProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@PlanDiagnosticsProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@StatisticsHealthProcedureObjectId, -2) AND (@QueryTextContains IS NULL OR qt.query_sql_text LIKE @QueryTextPattern ESCAPE N'~') ORDER BY + CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' AND aws.TotalWaitMilliseconds IS NULL THEN 1 ELSE 0 END ASC, CASE WHEN @OrderByNormalized = 'TotalDuration' THEN ars.TotalDurationMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'AverageDuration' THEN ars.AverageDurationMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'MaximumDuration' THEN ars.MaximumDurationMicroseconds END DESC, @@ -295,6 +378,7 @@ BEGIN CASE WHEN @OrderByNormalized = 'AverageCpu' THEN ars.AverageCpuMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'LogicalReads' THEN ars.TotalLogicalReads END DESC, CASE WHEN @OrderByNormalized = 'Executions' THEN ars.RegularExecutionCount END DESC, + CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) END DESC, q.query_id ASC, p.plan_id ASC OFFSET @Offset ROWS FETCH NEXT @Top ROWS ONLY; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql index 812c30fcf4..08c583d6c6 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql @@ -25,6 +25,8 @@ BEGIN DECLARE @QueryTextFilterLength int; DECLARE @QueryStoreState nvarchar(60); DECLARE @QueryStoreReadOnlyReason bigint; + DECLARE @WaitStatsCaptureMode nvarchar(60); + DECLARE @WaitStatsStatus varchar(32); DECLARE @SlowQueriesProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStoreSlowQueries'); DECLARE @PlanDiagnosticsProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStorePlanDiagnostics'); DECLARE @StatisticsHealthProcedureObjectId int = OBJECT_ID(N'dbo.GetStatisticsHealth'); @@ -48,9 +50,16 @@ BEGIN SELECT @QueryStoreState = actual_state_desc, - @QueryStoreReadOnlyReason = readonly_reason + @QueryStoreReadOnlyReason = readonly_reason, + @WaitStatsCaptureMode = wait_stats_capture_mode_desc FROM sys.database_query_store_options; + SET @WaitStatsStatus = + CASE @WaitStatsCaptureMode + WHEN N'ON' THEN 'Available' + WHEN N'OFF' THEN 'Disabled' + ELSE 'Unavailable' + END; SET @AuditText = CONCAT( @AuditText, N';StartTimeUtc=', CONVERT(nvarchar(33), @ResolvedStartTime, 127), @@ -62,7 +71,8 @@ BEGIN N';QueryTextFilterPresent=', CASE WHEN @QueryTextContains IS NULL THEN N'0' ELSE N'1' END, N';QueryTextFilterLength=', @QueryTextFilterLength, N';QueryStoreState=', ISNULL(@QueryStoreState, N'Unknown'), - N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown')); + N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown'), + N';WaitStatsCaptureMode=', ISNULL(@WaitStatsCaptureMode, N'Unknown')); EXECUTE dbo.LogEvent @Process = @ProcedureName, @@ -102,6 +112,7 @@ BEGIN WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageCpu' THEN 'AverageCpu' WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'LogicalReads' THEN 'LogicalReads' WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'Executions' THEN 'Executions' + WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalWait' THEN 'TotalWait' END; IF @OrderByNormalized IS NULL @@ -217,6 +228,63 @@ BEGIN FROM RankedCollapsedRuntimeStats AS rs GROUP BY rs.PlanId HAVING SUM(rs.RegularExecutionCount) >= @MinExecutions + ), + WaitStatsRows AS + ( + SELECT + ws.plan_id AS PlanId, + ws.wait_category_desc AS WaitCategoryDescription, + CONVERT(decimal(38, 0), ws.total_query_wait_time_ms) AS TotalWaitMilliseconds, + CONVERT(decimal(38, 0), ws.max_query_wait_time_ms) AS MaximumWaitMilliseconds + FROM sys.query_store_wait_stats AS ws + INNER JOIN sys.query_store_runtime_stats_interval AS rsi + ON rsi.runtime_stats_interval_id = ws.runtime_stats_interval_id + WHERE @WaitStatsStatus = 'Available' + AND ws.execution_type = 0 + AND rsi.start_time < @ResolvedEndTime + AND rsi.end_time > @ResolvedStartTime + ), + WaitCategories AS + ( + SELECT + ws.PlanId, + ws.WaitCategoryDescription, + CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds, + MAX(ws.MaximumWaitMilliseconds) AS MaximumWaitMilliseconds + FROM WaitStatsRows AS ws + GROUP BY + ws.PlanId, + ws.WaitCategoryDescription + ), + AggregatedWaitStats AS + ( + SELECT + ws.PlanId, + CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds + FROM WaitCategories AS ws + GROUP BY ws.PlanId + ), + WaitStatsXml AS + ( + SELECT + wp.PlanId, + ( + SELECT + wc.WaitCategoryDescription AS [@Category], + wc.TotalWaitMilliseconds AS [@TotalWaitMilliseconds], + CONVERT(decimal(38, 4), wc.TotalWaitMilliseconds / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) AS [@AverageWaitMilliseconds], + wc.MaximumWaitMilliseconds AS [@MaximumWaitMilliseconds] + FROM WaitCategories AS wc + WHERE wc.PlanId = wp.PlanId + AND wc.TotalWaitMilliseconds > 0 + ORDER BY + wc.TotalWaitMilliseconds DESC, + wc.WaitCategoryDescription ASC + FOR XML PATH(N'WaitCategory'), ROOT(N'WaitStats'), TYPE + ) AS WaitStatsXml + FROM (SELECT DISTINCT PlanId FROM WaitCategories) AS wp + INNER JOIN AggregatedRuntimeStats AS ars + ON ars.PlanId = wp.PlanId ) SELECT q.query_id AS QueryId, @@ -255,7 +323,17 @@ BEGIN p.compatibility_level AS CompatibilityLevel, p.is_online_index_plan AS IsOnlineIndexPlan, p.is_trivial_plan AS IsTrivialPlan, - p.is_parallel_plan AS IsParallelPlan + p.is_parallel_plan AS IsParallelPlan, + CASE + WHEN @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) + END AS TotalWaitMilliseconds, + CASE + WHEN @WaitStatsStatus = 'Available' THEN CONVERT(decimal(38, 4), ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) + END AS AverageWaitMilliseconds, + @WaitStatsStatus AS WaitStatsStatus, + CASE + WHEN @WaitStatsStatus = 'Available' THEN ISNULL(wsx.WaitStatsXml, CONVERT(xml, N'')) + END AS WaitStatsXml FROM AggregatedRuntimeStats AS ars INNER JOIN sys.query_store_plan AS p ON p.plan_id = ars.PlanId @@ -263,11 +341,16 @@ BEGIN ON q.query_id = p.query_id INNER JOIN sys.query_store_query_text AS qt ON qt.query_text_id = q.query_text_id + LEFT JOIN AggregatedWaitStats AS aws + ON aws.PlanId = ars.PlanId + LEFT JOIN WaitStatsXml AS wsx + ON wsx.PlanId = ars.PlanId WHERE ISNULL(q.object_id, -1) <> ISNULL(@SlowQueriesProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@PlanDiagnosticsProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@StatisticsHealthProcedureObjectId, -2) AND (@QueryTextContains IS NULL OR qt.query_sql_text LIKE @QueryTextPattern ESCAPE N'~') ORDER BY + CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' AND aws.TotalWaitMilliseconds IS NULL THEN 1 ELSE 0 END ASC, CASE WHEN @OrderByNormalized = 'TotalDuration' THEN ars.TotalDurationMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'AverageDuration' THEN ars.AverageDurationMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'MaximumDuration' THEN ars.MaximumDurationMicroseconds END DESC, @@ -275,6 +358,7 @@ BEGIN CASE WHEN @OrderByNormalized = 'AverageCpu' THEN ars.AverageCpuMicroseconds END DESC, CASE WHEN @OrderByNormalized = 'LogicalReads' THEN ars.TotalLogicalReads END DESC, CASE WHEN @OrderByNormalized = 'Executions' THEN ars.RegularExecutionCount END DESC, + CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) END DESC, q.query_id ASC, p.plan_id ASC OFFSET @Offset ROWS FETCH NEXT @Top ROWS ONLY; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs index 3ed5f639fc..884a40956b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs @@ -74,7 +74,6 @@ public async Task GivenAQueryStoreCapturedFhirQuery_WhenDiagnosticsProceduresAre await AssertPlanDiagnosticsAsync(connection, planId, queryMarker, CancellationToken.None); await AssertResourceStatisticsHealthAsync(connection, CancellationToken.None); - await AssertTotalWaitOrderingIsRejectedAsync(connection, CancellationToken.None); } private static async Task EnableAndVerifyQueryStoreAsync(SqlConnection connection, CancellationToken cancellationToken) @@ -147,7 +146,7 @@ private static async Task GetSlowQueryPlanIdAsync( command.Parameters.Add("@EndTime", SqlDbType.DateTimeOffset).Value = windowEnd; command.Parameters.Add("@Top", SqlDbType.Int).Value = 10; command.Parameters.Add("@Offset", SqlDbType.Int).Value = 0; - command.Parameters.Add("@OrderBy", SqlDbType.VarChar, 32).Value = "Executions"; + command.Parameters.Add("@OrderBy", SqlDbType.VarChar, 32).Value = "TotalWait"; command.Parameters.Add("@MinExecutions", SqlDbType.BigInt).Value = QueryExecutionCount; command.Parameters.Add("@QueryTextContains", SqlDbType.NVarChar, 256).Value = queryMarker; @@ -161,31 +160,35 @@ private static async Task GetSlowQueryPlanIdAsync( Assert.Contains(queryMarker, reader.GetString(reader.GetOrdinal("QuerySqlText"))); Assert.False(reader.IsDBNull(reader.GetOrdinal("FirstExecutionTimeUtc"))); Assert.False(reader.IsDBNull(reader.GetOrdinal("LastExecutionTimeUtc"))); - AssertWaitColumnsAreNotReturned(reader); + AssertWaitStatisticsAvailability(reader); Assert.False(await reader.ReadAsync(cancellationToken), "The unique query-text filter returned more than one Query Store plan."); Assert.False(await reader.NextResultAsync(cancellationToken), "dbo.GetQueryStoreSlowQueries returned more than one result set."); return planId; } - private static void AssertWaitColumnsAreNotReturned(SqlDataReader reader) + private static void AssertWaitStatisticsAvailability(SqlDataReader reader) { - Assert.Throws(() => reader.GetOrdinal("TotalWaitMilliseconds")); - Assert.Throws(() => reader.GetOrdinal("AverageWaitMilliseconds")); - Assert.Throws(() => reader.GetOrdinal("WaitStatsStatus")); - Assert.Throws(() => reader.GetOrdinal("WaitStatsXml")); - } + int totalWaitMillisecondsOrdinal = reader.GetOrdinal("TotalWaitMilliseconds"); + int averageWaitMillisecondsOrdinal = reader.GetOrdinal("AverageWaitMilliseconds"); + int waitStatsStatusOrdinal = reader.GetOrdinal("WaitStatsStatus"); + int waitStatsXmlOrdinal = reader.GetOrdinal("WaitStatsXml"); + string waitStatsStatus = reader.GetString(waitStatsStatusOrdinal); - private static async Task AssertTotalWaitOrderingIsRejectedAsync(SqlConnection connection, CancellationToken cancellationToken) - { - using SqlCommand command = connection.CreateCommand(); - command.CommandType = CommandType.StoredProcedure; - command.CommandText = "dbo.GetQueryStoreSlowQueries"; - command.Parameters.Add("@OrderBy", SqlDbType.VarChar, 32).Value = "TotalWait"; + Assert.Contains(waitStatsStatus, new[] { "Available", "Disabled", "Unavailable" }); - SqlException exception = await Assert.ThrowsAsync(() => command.ExecuteNonQueryAsync(cancellationToken)); - - Assert.Contains("@OrderBy is not supported.", exception.Message); + if (string.Equals(waitStatsStatus, "Available", StringComparison.Ordinal)) + { + Assert.False(reader.IsDBNull(totalWaitMillisecondsOrdinal)); + Assert.False(reader.IsDBNull(averageWaitMillisecondsOrdinal)); + Assert.False(reader.IsDBNull(waitStatsXmlOrdinal)); + } + else + { + Assert.True(reader.IsDBNull(totalWaitMillisecondsOrdinal)); + Assert.True(reader.IsDBNull(averageWaitMillisecondsOrdinal)); + Assert.True(reader.IsDBNull(waitStatsXmlOrdinal)); + } } private static async Task AssertPlanDiagnosticsAsync( From 94c8ea02d3cb25739c49eea0000f08c1ec0d8082 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 18 Aug 2026 16:46:23 +0000 Subject: [PATCH 08/20] Clarify Query Store diagnostic SQL Add section headers and focused rationale for validation, aggregation, edge cases, plan sanitization, and error handling without changing executable SQL.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Features/Schema/Migrations/117.diff.sql | 90 +++++++++++++++++++ .../Sprocs/GetQueryStorePlanDiagnostics.sql | 24 +++++ .../Sql/Sprocs/GetQueryStoreSlowQueries.sql | 38 ++++++++ .../Schema/Sql/Sprocs/GetStatisticsHealth.sql | 28 ++++++ 4 files changed, 180 insertions(+) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql index 8f51714398..52b7715346 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql @@ -59,6 +59,9 @@ BEGIN N';EffectivePrincipal=', USER_NAME()); BEGIN TRY + -- ------------------------------------------------------------------------ + -- Resolve defaults, capture Query Store state, and build the audit context + -- ------------------------------------------------------------------------ SET @Top = ISNULL(@Top, 20); SET @Offset = ISNULL(@Offset, 0); SET @MinExecutions = ISNULL(@MinExecutions, 1); @@ -100,6 +103,9 @@ BEGIN @Status = 'Start', @Text = @AuditText; + -- ------------------------------------------------------------------------ + -- Validate parameters + -- ------------------------------------------------------------------------ IF @Top < 1 OR @Top > 100 THROW 50400, '@Top must be between 1 and 100.', 1; @@ -138,15 +144,28 @@ BEGIN IF @OrderByNormalized IS NULL THROW 50407, '@OrderBy is not supported.', 1; + -- ------------------------------------------------------------------------ + -- Validate Query Store prerequisite state + -- ------------------------------------------------------------------------ IF ISNULL(@QueryStoreState, N'') NOT IN (N'READ_WRITE', N'READ_ONLY') THROW 50408, 'Query Store is not enabled and readable.', 1; + -- ------------------------------------------------------------------------ + -- Normalize the literal filter into a safe LIKE pattern + -- ------------------------------------------------------------------------ + -- Escape wildcard/escape characters before wrapping so a literal '%', '_', or '[' in the + -- caller-supplied text is matched literally rather than interpreted by LIKE. SET @QueryTextPattern = CASE WHEN @QueryTextContains IS NULL THEN NULL ELSE N'%' + REPLACE(REPLACE(REPLACE(REPLACE(@QueryTextContains, N'~', N'~~'), N'%', N'~%'), N'_', N'~_'), N'[', N'~[') + N'%' END; + -- ------------------------------------------------------------------------ + -- Collapse duplicate runtime-stats rows and compute weighted aggregates + -- ------------------------------------------------------------------------ + -- Active Query Store intervals can expose both a persisted row and an in-memory row for the + -- same plan/interval, so duplicates must be collapsed before aggregating across intervals. ;WITH RuntimeStatsRows AS ( SELECT @@ -172,6 +191,8 @@ BEGIN ON rsi.runtime_stats_interval_id = rs.runtime_stats_interval_id -- The baseline is regular executions only. An explicit execution-type input belongs here if added later. WHERE rs.execution_type = 0 + -- Query Store interval overlap semantics are inclusive of edge executions, so a run that only + -- partially overlaps the requested [@ResolvedStartTime, @ResolvedEndTime) window is still included. AND rsi.start_time < @ResolvedEndTime AND rsi.end_time > @ResolvedStartTime ), @@ -229,6 +250,8 @@ BEGIN rs.PlanId, SUM(rs.RegularExecutionCount) AS RegularExecutionCount, CONVERT(decimal(38, 0), SUM(rs.TotalDurationMicroseconds)) AS TotalDurationMicroseconds, + -- Averages are recomputed as execution-count-weighted sums rather than averaged directly, + -- and division uses decimal precision so intervals with unequal execution counts are not skewed. CONVERT(decimal(38, 4), SUM(rs.TotalDurationMicroseconds) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageDurationMicroseconds, MIN(rs.MinimumDurationMicroseconds) AS MinimumDurationMicroseconds, MAX(rs.MaximumDurationMicroseconds) AS MaximumDurationMicroseconds, @@ -249,6 +272,9 @@ BEGIN GROUP BY rs.PlanId HAVING SUM(rs.RegularExecutionCount) >= @MinExecutions ), + -- ------------------------------------------------------------------------ + -- Aggregate wait-stat rows and handle capture availability + -- ------------------------------------------------------------------------ WaitStatsRows AS ( SELECT @@ -259,6 +285,8 @@ BEGIN FROM sys.query_store_wait_stats AS ws INNER JOIN sys.query_store_runtime_stats_interval AS rsi ON rsi.runtime_stats_interval_id = ws.runtime_stats_interval_id + -- When wait capture is disabled or unavailable this CTE is intentionally left empty so runtime + -- rows are still returned with NULL wait columns instead of failing the whole query. WHERE @WaitStatsStatus = 'Available' AND ws.execution_type = 0 AND rsi.start_time < @ResolvedEndTime @@ -306,6 +334,9 @@ BEGIN INNER JOIN AggregatedRuntimeStats AS ars ON ars.PlanId = wp.PlanId ) + -- ------------------------------------------------------------------------ + -- Project results with static, deterministic ordering + -- ------------------------------------------------------------------------ SELECT q.query_id AS QueryId, p.plan_id AS PlanId, @@ -365,10 +396,14 @@ BEGIN ON aws.PlanId = ars.PlanId LEFT JOIN WaitStatsXml AS wsx ON wsx.PlanId = ars.PlanId + -- Self-exclusion keeps these diagnostic procedures' own Query Store entries out of their own + -- results, so running diagnostics does not appear as a "slow query" in the output. WHERE ISNULL(q.object_id, -1) <> ISNULL(@SlowQueriesProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@PlanDiagnosticsProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@StatisticsHealthProcedureObjectId, -2) AND (@QueryTextContains IS NULL OR qt.query_sql_text LIKE @QueryTextPattern ESCAPE N'~') + -- The requested @OrderBy column drives the primary sort key and every other CASE branch + -- evaluates to NULL, so ties still fall back to query_id/plan_id for a stable, deterministic order. ORDER BY CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' AND aws.TotalWaitMilliseconds IS NULL THEN 1 ELSE 0 END ASC, CASE WHEN @OrderByNormalized = 'TotalDuration' THEN ars.TotalDurationMicroseconds END DESC, @@ -394,6 +429,9 @@ BEGIN @Text = @AuditText; END TRY BEGIN CATCH + -- ------------------------------------------------------------------------ + -- Audit the failure and rethrow + -- ------------------------------------------------------------------------ SET @AuditText = CONCAT( @AuditText, N';ErrorNumber=', ERROR_NUMBER(), @@ -467,6 +505,9 @@ BEGIN BEGIN TRY EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Start',@Text=@AuditText + -- ------------------------------------------------------------------------ + -- Validate parameters and prerequisite Query Store state + -- ------------------------------------------------------------------------ IF @PlanId IS NULL OR @PlanId <= 0 THROW 50001, 'Plan ID must be a positive bigint.', 1 @@ -476,6 +517,11 @@ BEGIN IF @QueryStoreState IS NULL OR @QueryStoreState NOT IN ('READ_WRITE', 'READ_ONLY') THROW 50002, 'Query Store is not readable.', 1 + -- ------------------------------------------------------------------------ + -- Look up the plan and collect metadata safely into local variables + -- ------------------------------------------------------------------------ + -- Projecting into scalar variables (rather than selecting directly) keeps the raw, unsanitized + -- query_plan XML out of the result set contract while it is still being sanitized below. SELECT @FoundPlanId = p.plan_id ,@QueryId = p.query_id ,@QueryHash = q.query_hash @@ -514,6 +560,11 @@ BEGIN FROM sys.query_store_runtime_stats WHERE plan_id = @PlanId + -- ------------------------------------------------------------------------ + -- Sanitize the Showplan XML and verify the ParameterList removal + -- ------------------------------------------------------------------------ + -- Raw or partially sanitized plans are never returned to the caller: this branch either produces + -- a verified-clean plan or leaves @SanitizedShowPlanXml NULL with a status/error code explaining why. IF @RawQueryPlan IS NULL BEGIN SET @SanitizationStatus = 'PlanXmlUnavailable' @@ -532,8 +583,13 @@ BEGIN BEGIN BEGIN TRY SET @SanitizedShowPlanXml = @LocalPlanXml + -- local-name() matches the ParameterList element regardless of the Showplan XML + -- namespace/version, since Query Store XML namespaces can vary across engine versions. SET @SanitizedShowPlanXml.modify('delete //*[local-name(.) = "ParameterList"]') + -- Defense-in-depth verification: check both the structural XML (no remaining + -- ParameterList elements/attributes) and the serialized text (no leftover literal + -- tokens) before trusting the sanitized plan is safe to return. SET @RemainingParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') SET @ForbiddenAttributeCount = @SanitizedShowPlanXml.value('count(//@*[local-name(.) = "ParameterCompiledValue" or local-name(.) = "ParameterRuntimeValue"])', 'bigint') SET @SerializedPlanXml = CONVERT(nvarchar(max), @SanitizedShowPlanXml) @@ -575,6 +631,9 @@ BEGIN SET @Rows = 1 + -- ------------------------------------------------------------------------ + -- Project the plan metadata and verified Showplan XML + -- ------------------------------------------------------------------------ SELECT @FoundPlanId AS PlanId ,@QueryId AS QueryId ,@QueryHash AS QueryHash @@ -607,6 +666,9 @@ BEGIN BEGIN CATCH SET @CaughtErrorNumber = ERROR_NUMBER() SET @CaughtErrorState = ERROR_STATE() + -- ------------------------------------------------------------------------ + -- Audit the failure and rethrow + -- ------------------------------------------------------------------------ SET @AuditText = CONCAT( N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), @@ -651,6 +713,9 @@ BEGIN BEGIN TRY EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @AuditText; + -- ------------------------------------------------------------------------ + -- Validate parameters + -- ------------------------------------------------------------------------ IF @Top IS NULL OR @Top < 1 OR @Top > 100 BEGIN THROW 50000, '@Top must be between 1 and 100.', 127; @@ -674,6 +739,9 @@ BEGIN THROW 50000, '@TableName must be nonblank when supplied.', 127; END + -- ------------------------------------------------------------------------ + -- Resolve the requested table name to exactly one user table + -- ------------------------------------------------------------------------ SELECT @TableCount = COUNT(*), @TableObjectId = MIN(tableInfo.object_id) @@ -697,6 +765,9 @@ BEGIN N';TableName=', ISNULL(@TableName, N'NULL'), N';OrderBy=', @NormalizedOrderBy); + -- ------------------------------------------------------------------------ + -- Project statistics-column metadata as XML alongside stats/index properties + -- ------------------------------------------------------------------------ ;WITH StatisticsMetadata AS ( SELECT @@ -735,6 +806,9 @@ BEGIN statisticsProperties.rows AS [Rows], statisticsProperties.unfiltered_rows AS UnfilteredRows, statisticsProperties.rows_sampled AS RowsSampled, + -- NULLIF guards a zero-row denominator (returns NULL instead of a divide-by-zero error), and + -- a percentage can legitimately exceed 100 (e.g. rows_sampled/modification_counter can outgrow + -- a stale rows count), so the result is not clamped. CONVERT(decimal(38, 4), (CONVERT(decimal(38, 0), statisticsProperties.rows_sampled) * CONVERT(decimal(3, 0), 100)) / NULLIF(CONVERT(decimal(38, 0), statisticsProperties.rows), CONVERT(decimal(38, 0), 0))) AS SamplingPercent, @@ -753,6 +827,9 @@ BEGIN LEFT JOIN sys.indexes AS indexInfo ON indexInfo.object_id = statisticsInfo.object_id AND indexInfo.index_id = statisticsInfo.stats_id + -- OUTER APPLY (rather than CROSS APPLY) preserves the statistics row even when + -- sys.dm_db_stats_properties returns nothing, e.g. for an unsupported/inaccessible object; + -- StatisticsStatus below reports 'PropertiesUnavailable' instead of silently dropping the row. OUTER APPLY ( SELECT @@ -769,6 +846,8 @@ BEGIN AND ( (@TableName IS NOT NULL AND tableInfo.object_id = @TableObjectId) + -- Without an explicit @TableName, temporal history tables (temporal_type = 1) are + -- excluded because their statistics mirror the corresponding current table. OR (@TableName IS NULL AND tableInfo.temporal_type <> 1) ) @@ -782,18 +861,25 @@ BEGIN ORDER BY CASE WHEN @NormalizedOrderBy = 'MODIFICATIONCOUNT' THEN ModificationCount END DESC, CASE WHEN @NormalizedOrderBy = 'MODIFICATIONPERCENT' THEN ModificationPercent END DESC, + -- NULL LastUpdated (properties unavailable) sorts first regardless of ASC/DESC, + -- then the actual timestamp orders the known values oldest-first. CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' AND LastUpdated IS NULL THEN 0 WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN 1 END ASC, CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN LastUpdated END ASC, CASE WHEN @NormalizedOrderBy = 'SAMPLINGPERCENT' THEN SamplingPercent END DESC, CASE WHEN @NormalizedOrderBy = 'ROWS' THEN [Rows] END DESC, + -- Table/statistics identity is the final, always-present tie-breaker so paging + -- is deterministic regardless of which @OrderBy column is requested. TableName ASC, StatisticsName ASC, StatisticsId ASC ) AS RowNumber FROM StatisticsMetadata ) + -- ------------------------------------------------------------------------ + -- Project results and apply offset/top paging over the deterministic order + -- ------------------------------------------------------------------------ SELECT TableName, StatisticsName, @@ -833,6 +919,9 @@ BEGIN BEGIN CATCH SET @CaughtErrorNumber = ERROR_NUMBER(); SET @CaughtErrorState = ERROR_STATE(); + -- ------------------------------------------------------------------------ + -- Audit the failure and rethrow + -- ------------------------------------------------------------------------ SET @AuditText = CONCAT( N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), @@ -840,6 +929,7 @@ BEGIN N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)); + -- Real error is before 1750, cannot trap in SQL; rethrow immediately without attempting to audit. IF ERROR_NUMBER() = 1750 THROW; EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @Start, @Text = @AuditText; THROW; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql index a1e2016db3..d248bfbc2d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql @@ -56,6 +56,9 @@ BEGIN BEGIN TRY EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Start',@Text=@AuditText + -- ------------------------------------------------------------------------ + -- Validate parameters and prerequisite Query Store state + -- ------------------------------------------------------------------------ IF @PlanId IS NULL OR @PlanId <= 0 THROW 50001, 'Plan ID must be a positive bigint.', 1 @@ -65,6 +68,11 @@ BEGIN IF @QueryStoreState IS NULL OR @QueryStoreState NOT IN ('READ_WRITE', 'READ_ONLY') THROW 50002, 'Query Store is not readable.', 1 + -- ------------------------------------------------------------------------ + -- Look up the plan and collect metadata safely into local variables + -- ------------------------------------------------------------------------ + -- Projecting into scalar variables (rather than selecting directly) keeps the raw, unsanitized + -- query_plan XML out of the result set contract while it is still being sanitized below. SELECT @FoundPlanId = p.plan_id ,@QueryId = p.query_id ,@QueryHash = q.query_hash @@ -103,6 +111,11 @@ BEGIN FROM sys.query_store_runtime_stats WHERE plan_id = @PlanId + -- ------------------------------------------------------------------------ + -- Sanitize the Showplan XML and verify the ParameterList removal + -- ------------------------------------------------------------------------ + -- Raw or partially sanitized plans are never returned to the caller: this branch either produces + -- a verified-clean plan or leaves @SanitizedShowPlanXml NULL with a status/error code explaining why. IF @RawQueryPlan IS NULL BEGIN SET @SanitizationStatus = 'PlanXmlUnavailable' @@ -121,8 +134,13 @@ BEGIN BEGIN BEGIN TRY SET @SanitizedShowPlanXml = @LocalPlanXml + -- local-name() matches the ParameterList element regardless of the Showplan XML + -- namespace/version, since Query Store XML namespaces can vary across engine versions. SET @SanitizedShowPlanXml.modify('delete //*[local-name(.) = "ParameterList"]') + -- Defense-in-depth verification: check both the structural XML (no remaining + -- ParameterList elements/attributes) and the serialized text (no leftover literal + -- tokens) before trusting the sanitized plan is safe to return. SET @RemainingParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') SET @ForbiddenAttributeCount = @SanitizedShowPlanXml.value('count(//@*[local-name(.) = "ParameterCompiledValue" or local-name(.) = "ParameterRuntimeValue"])', 'bigint') SET @SerializedPlanXml = CONVERT(nvarchar(max), @SanitizedShowPlanXml) @@ -164,6 +182,9 @@ BEGIN SET @Rows = 1 + -- ------------------------------------------------------------------------ + -- Project the plan metadata and verified Showplan XML + -- ------------------------------------------------------------------------ SELECT @FoundPlanId AS PlanId ,@QueryId AS QueryId ,@QueryHash AS QueryHash @@ -196,6 +217,9 @@ BEGIN BEGIN CATCH SET @CaughtErrorNumber = ERROR_NUMBER() SET @CaughtErrorState = ERROR_STATE() + -- ------------------------------------------------------------------------ + -- Audit the failure and rethrow + -- ------------------------------------------------------------------------ SET @AuditText = CONCAT( N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql index 08c583d6c6..7a3919ffaa 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql @@ -39,6 +39,9 @@ BEGIN N';EffectivePrincipal=', USER_NAME()); BEGIN TRY + -- ------------------------------------------------------------------------ + -- Resolve defaults, capture Query Store state, and build the audit context + -- ------------------------------------------------------------------------ SET @Top = ISNULL(@Top, 20); SET @Offset = ISNULL(@Offset, 0); SET @MinExecutions = ISNULL(@MinExecutions, 1); @@ -80,6 +83,9 @@ BEGIN @Status = 'Start', @Text = @AuditText; + -- ------------------------------------------------------------------------ + -- Validate parameters + -- ------------------------------------------------------------------------ IF @Top < 1 OR @Top > 100 THROW 50400, '@Top must be between 1 and 100.', 1; @@ -118,15 +124,28 @@ BEGIN IF @OrderByNormalized IS NULL THROW 50407, '@OrderBy is not supported.', 1; + -- ------------------------------------------------------------------------ + -- Validate Query Store prerequisite state + -- ------------------------------------------------------------------------ IF ISNULL(@QueryStoreState, N'') NOT IN (N'READ_WRITE', N'READ_ONLY') THROW 50408, 'Query Store is not enabled and readable.', 1; + -- ------------------------------------------------------------------------ + -- Normalize the literal filter into a safe LIKE pattern + -- ------------------------------------------------------------------------ + -- Escape wildcard/escape characters before wrapping so a literal '%', '_', or '[' in the + -- caller-supplied text is matched literally rather than interpreted by LIKE. SET @QueryTextPattern = CASE WHEN @QueryTextContains IS NULL THEN NULL ELSE N'%' + REPLACE(REPLACE(REPLACE(REPLACE(@QueryTextContains, N'~', N'~~'), N'%', N'~%'), N'_', N'~_'), N'[', N'~[') + N'%' END; + -- ------------------------------------------------------------------------ + -- Collapse duplicate runtime-stats rows and compute weighted aggregates + -- ------------------------------------------------------------------------ + -- Active Query Store intervals can expose both a persisted row and an in-memory row for the + -- same plan/interval, so duplicates must be collapsed before aggregating across intervals. ;WITH RuntimeStatsRows AS ( SELECT @@ -152,6 +171,8 @@ BEGIN ON rsi.runtime_stats_interval_id = rs.runtime_stats_interval_id -- The baseline is regular executions only. An explicit execution-type input belongs here if added later. WHERE rs.execution_type = 0 + -- Query Store interval overlap semantics are inclusive of edge executions, so a run that only + -- partially overlaps the requested [@ResolvedStartTime, @ResolvedEndTime) window is still included. AND rsi.start_time < @ResolvedEndTime AND rsi.end_time > @ResolvedStartTime ), @@ -209,6 +230,8 @@ BEGIN rs.PlanId, SUM(rs.RegularExecutionCount) AS RegularExecutionCount, CONVERT(decimal(38, 0), SUM(rs.TotalDurationMicroseconds)) AS TotalDurationMicroseconds, + -- Averages are recomputed as execution-count-weighted sums rather than averaged directly, + -- and division uses decimal precision so intervals with unequal execution counts are not skewed. CONVERT(decimal(38, 4), SUM(rs.TotalDurationMicroseconds) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageDurationMicroseconds, MIN(rs.MinimumDurationMicroseconds) AS MinimumDurationMicroseconds, MAX(rs.MaximumDurationMicroseconds) AS MaximumDurationMicroseconds, @@ -229,6 +252,9 @@ BEGIN GROUP BY rs.PlanId HAVING SUM(rs.RegularExecutionCount) >= @MinExecutions ), + -- ------------------------------------------------------------------------ + -- Aggregate wait-stat rows and handle capture availability + -- ------------------------------------------------------------------------ WaitStatsRows AS ( SELECT @@ -239,6 +265,8 @@ BEGIN FROM sys.query_store_wait_stats AS ws INNER JOIN sys.query_store_runtime_stats_interval AS rsi ON rsi.runtime_stats_interval_id = ws.runtime_stats_interval_id + -- When wait capture is disabled or unavailable this CTE is intentionally left empty so runtime + -- rows are still returned with NULL wait columns instead of failing the whole query. WHERE @WaitStatsStatus = 'Available' AND ws.execution_type = 0 AND rsi.start_time < @ResolvedEndTime @@ -286,6 +314,9 @@ BEGIN INNER JOIN AggregatedRuntimeStats AS ars ON ars.PlanId = wp.PlanId ) + -- ------------------------------------------------------------------------ + -- Project results with static, deterministic ordering + -- ------------------------------------------------------------------------ SELECT q.query_id AS QueryId, p.plan_id AS PlanId, @@ -345,10 +376,14 @@ BEGIN ON aws.PlanId = ars.PlanId LEFT JOIN WaitStatsXml AS wsx ON wsx.PlanId = ars.PlanId + -- Self-exclusion keeps these diagnostic procedures' own Query Store entries out of their own + -- results, so running diagnostics does not appear as a "slow query" in the output. WHERE ISNULL(q.object_id, -1) <> ISNULL(@SlowQueriesProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@PlanDiagnosticsProcedureObjectId, -2) AND ISNULL(q.object_id, -1) <> ISNULL(@StatisticsHealthProcedureObjectId, -2) AND (@QueryTextContains IS NULL OR qt.query_sql_text LIKE @QueryTextPattern ESCAPE N'~') + -- The requested @OrderBy column drives the primary sort key and every other CASE branch + -- evaluates to NULL, so ties still fall back to query_id/plan_id for a stable, deterministic order. ORDER BY CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' AND aws.TotalWaitMilliseconds IS NULL THEN 1 ELSE 0 END ASC, CASE WHEN @OrderByNormalized = 'TotalDuration' THEN ars.TotalDurationMicroseconds END DESC, @@ -374,6 +409,9 @@ BEGIN @Text = @AuditText; END TRY BEGIN CATCH + -- ------------------------------------------------------------------------ + -- Audit the failure and rethrow + -- ------------------------------------------------------------------------ SET @AuditText = CONCAT( @AuditText, N';ErrorNumber=', ERROR_NUMBER(), diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql index 2df69dff55..407d12aa2f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql @@ -27,6 +27,9 @@ BEGIN BEGIN TRY EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @AuditText; + -- ------------------------------------------------------------------------ + -- Validate parameters + -- ------------------------------------------------------------------------ IF @Top IS NULL OR @Top < 1 OR @Top > 100 BEGIN THROW 50000, '@Top must be between 1 and 100.', 127; @@ -50,6 +53,9 @@ BEGIN THROW 50000, '@TableName must be nonblank when supplied.', 127; END + -- ------------------------------------------------------------------------ + -- Resolve the requested table name to exactly one user table + -- ------------------------------------------------------------------------ SELECT @TableCount = COUNT(*), @TableObjectId = MIN(tableInfo.object_id) @@ -73,6 +79,9 @@ BEGIN N';TableName=', ISNULL(@TableName, N'NULL'), N';OrderBy=', @NormalizedOrderBy); + -- ------------------------------------------------------------------------ + -- Project statistics-column metadata as XML alongside stats/index properties + -- ------------------------------------------------------------------------ ;WITH StatisticsMetadata AS ( SELECT @@ -111,6 +120,9 @@ BEGIN statisticsProperties.rows AS [Rows], statisticsProperties.unfiltered_rows AS UnfilteredRows, statisticsProperties.rows_sampled AS RowsSampled, + -- NULLIF guards a zero-row denominator (returns NULL instead of a divide-by-zero error), and + -- a percentage can legitimately exceed 100 (e.g. rows_sampled/modification_counter can outgrow + -- a stale rows count), so the result is not clamped. CONVERT(decimal(38, 4), (CONVERT(decimal(38, 0), statisticsProperties.rows_sampled) * CONVERT(decimal(3, 0), 100)) / NULLIF(CONVERT(decimal(38, 0), statisticsProperties.rows), CONVERT(decimal(38, 0), 0))) AS SamplingPercent, @@ -129,6 +141,9 @@ BEGIN LEFT JOIN sys.indexes AS indexInfo ON indexInfo.object_id = statisticsInfo.object_id AND indexInfo.index_id = statisticsInfo.stats_id + -- OUTER APPLY (rather than CROSS APPLY) preserves the statistics row even when + -- sys.dm_db_stats_properties returns nothing, e.g. for an unsupported/inaccessible object; + -- StatisticsStatus below reports 'PropertiesUnavailable' instead of silently dropping the row. OUTER APPLY ( SELECT @@ -145,6 +160,8 @@ BEGIN AND ( (@TableName IS NOT NULL AND tableInfo.object_id = @TableObjectId) + -- Without an explicit @TableName, temporal history tables (temporal_type = 1) are + -- excluded because their statistics mirror the corresponding current table. OR (@TableName IS NULL AND tableInfo.temporal_type <> 1) ) @@ -158,18 +175,25 @@ BEGIN ORDER BY CASE WHEN @NormalizedOrderBy = 'MODIFICATIONCOUNT' THEN ModificationCount END DESC, CASE WHEN @NormalizedOrderBy = 'MODIFICATIONPERCENT' THEN ModificationPercent END DESC, + -- NULL LastUpdated (properties unavailable) sorts first regardless of ASC/DESC, + -- then the actual timestamp orders the known values oldest-first. CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' AND LastUpdated IS NULL THEN 0 WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN 1 END ASC, CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN LastUpdated END ASC, CASE WHEN @NormalizedOrderBy = 'SAMPLINGPERCENT' THEN SamplingPercent END DESC, CASE WHEN @NormalizedOrderBy = 'ROWS' THEN [Rows] END DESC, + -- Table/statistics identity is the final, always-present tie-breaker so paging + -- is deterministic regardless of which @OrderBy column is requested. TableName ASC, StatisticsName ASC, StatisticsId ASC ) AS RowNumber FROM StatisticsMetadata ) + -- ------------------------------------------------------------------------ + -- Project results and apply offset/top paging over the deterministic order + -- ------------------------------------------------------------------------ SELECT TableName, StatisticsName, @@ -209,6 +233,9 @@ BEGIN BEGIN CATCH SET @CaughtErrorNumber = ERROR_NUMBER(); SET @CaughtErrorState = ERROR_STATE(); + -- ------------------------------------------------------------------------ + -- Audit the failure and rethrow + -- ------------------------------------------------------------------------ SET @AuditText = CONCAT( N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), @@ -216,6 +243,7 @@ BEGIN N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)); + -- Real error is before 1750, cannot trap in SQL; rethrow immediately without attempting to audit. IF ERROR_NUMBER() = 1750 THROW; EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @Start, @Text = @AuditText; THROW; From 5b883be80665ce29953e9a195633a640697d2109 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 00:49:24 +0000 Subject: [PATCH 09/20] Redesign Query Store diagnostics as an opt-in push-based watchdog Replaces the stored-procedure design with an opt-in background watchdog that runs inside the FHIR server on its existing SQL identity and pushes diagnostics out as IMetricsNotification messages. The previous design exposed three stored procedures behind a new FhirDiagnosticsReader execute-only role, to be called by an external operational principal. That introduced an entirely new inbound permission model the service does not otherwise have. This version inverts the direction: nothing connects in, the server emits out. Changes: - QueryStoreDiagnosticsWatchdog reads Query Store on a configurable period using the existing Watchdog lease, so exactly one instance runs per database. - Detect-only Query Store handling: the watchdog reports state and skips its cycle when unavailable, and never issues ALTER DATABASE. - Plan sanitization moved from 236 lines of T-SQL into C# (QueryPlanSanitizer), which is more reliable and testable. - Emits SlowQueryNotification, QueryPlanNotification and StatisticsHealthNotification. - Disabled by default. Enabled via FhirServer:Watchdog:QueryStoreDiagnostics and gated at runtime by the dbo.Parameters row, matching DefragWatchdog. No schema change: all SQL is inline, so schema version stays at V116 and the migration, role script and sprocs are removed. Tests: 9 sanitizer unit tests, plus 3 integration tests that execute every inline statement against a live SQL Server and assert all three notifications publish. AB#186447 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 589 +++-------- nuget.config | 9 +- .../QueryStoreDiagnosticsConfiguration.cs | 49 + .../Configs/WatchdogConfiguration.cs | 5 + .../Features/Metrics/QueryPlanNotification.cs | 67 ++ .../Features/Metrics/SlowQueryNotification.cs | 121 +++ .../Metrics/StatisticsHealthNotification.cs | 90 ++ .../appsettings.json | 9 + .../Watchdogs/QueryPlanSanitizerTests.cs | 225 +++++ .../Features/Schema/Migrations/117.diff.sql | 944 ------------------ .../Features/Schema/SchemaVersion.cs | 1 - .../Features/Schema/SchemaVersionConstants.cs | 2 +- .../Schema/Sql/Scripts/DiagnosticsRole.sql | 26 - .../Sprocs/GetQueryStorePlanDiagnostics.sql | 236 ----- .../Sql/Sprocs/GetQueryStoreSlowQueries.sql | 430 -------- .../Schema/Sql/Sprocs/GetStatisticsHealth.sql | 252 ----- .../Watchdogs/QueryPlanSanitizationResult.cs | 36 + .../Features/Watchdogs/QueryPlanSanitizer.cs | 85 ++ .../QueryStoreDiagnosticsWatchdog.cs | 594 +++++++++++ .../Watchdogs/WatchdogsBackgroundService.cs | 8 + .../Microsoft.Health.Fhir.SqlServer.csproj | 3 +- ...rBuilderSqlServerRegistrationExtensions.cs | 1 + ...th.Fhir.Shared.Tests.Integration.projitems | 2 +- .../QueryStoreDiagnosticsWatchdogTests.cs | 279 ++++++ .../SqlServerQueryStoreDiagnosticsTests.cs | 264 ----- 25 files changed, 1720 insertions(+), 2607 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Metrics/QueryPlanNotification.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Metrics/StatisticsHealthNotification.cs create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs delete mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql delete mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/DiagnosticsRole.sql delete mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql delete mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql delete mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs create mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs delete mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index ba4353c9bc..2e4da90741 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -2,20 +2,20 @@ ## Status -Agreed baseline for implementation. This document defines the SQL contract, security boundary, operational limits, validation requirements, and repository ownership split. It does not select a Geneva action, PaaS API, or direct-SQL caller. +Agreed baseline for implementation. This document defines an opt-in background worker that emits Azure SQL Query Store diagnostics from inside the FHIR server, the enablement model, the emitted contracts, the disclosure boundary, and the repository ownership split. + +This specification **supersedes an earlier pull-based design** in which three stored procedures were called by an external caller. See [Rejected alternative](#rejected-alternative-caller-invoked-stored-procedures). ## Problem -FHIR Azure SQL performance investigations currently require privileged, manual access to Query Store plans, runtime metrics, wait statistics, and statistics metadata. Although Azure SQL can export `QueryStoreWaitStatistics` to Log Analytics, the SQL diagnostics intentionally include plan-level waits so direct SQL and future Geneva callers receive one self-contained slow-query result with runtime metrics, waits, query text, and plan IDs without joining Log Analytics. Support engineers still need a bounded way to: +FHIR Azure SQL performance investigations currently require privileged, manual access to Query Store plans, runtime metrics, wait statistics, and statistics metadata. Support engineers need a bounded way to: - identify expensive or regressed query plans; -- retrieve an SSMS-viewable Query Store Showplan; +- obtain an SSMS-viewable Showplan for a slow plan; - compare runtime and wait metrics; and - inspect statistics freshness, sampling, and cardinality metadata. -The baseline is a self-contained, read-only SQL interface. Filtering, validation, redaction, paging, permissions, and auditing must live in SQL so the procedures can be used by an authorized direct SQL connection or wrapped by future operational tooling. - -The canonical database objects belong to the OSS `fhir-server` schema because that repository owns the versioned SQL migration chain consumed by FHIR PaaS. PaaS owns how authorized operators invoke the contract and handle its results. +Azure SQL diagnostic settings can already export `QueryStoreRuntimeStatistics`, `QueryStoreWaitStatistics`, and `AutomaticTuning` to Log Analytics. What that stream does **not** carry is the Query Store **query text** and the **Showplan XML**, which are the two artifacts an investigation actually needs in order to reason about a regression. This feature closes that gap. Related internal guidance: @@ -24,526 +24,227 @@ Related internal guidance: - `Health.wiki/Home/Olympus-Team/DRI/TSGs/SQL-Latency-Issues/SQL-Statistics-Overview.md` - `Health.wiki/Home/Olympus-Team/Development/SQL-Performance-Automation.md` -## Goals - -1. Identify slow or resource-intensive query plans over a bounded time range. -2. Return full Query Store query text to authorized diagnostic callers. -3. Return an SSMS-viewable Query Store Showplan after removing parameter-value metadata. -4. Include Query Store wait statistics in slow-query results when capture is available. -5. Report statistics freshness, sampling, and filter metadata without returning histogram values. -6. Provide an execute-only database role for least-privilege callers. -7. Keep the SQL contract independent of any API, Geneva action, or other caller implementation. - -## Non-goals - -- Retrieving or capturing an actual execution plan. -- Reconstructing or executing SQL from Query Store. -- Returning statistics histograms, density vectors, or sampled column values. -- Clearing the procedure cache, updating statistics, forcing plans, or changing Query Store configuration. -- Providing caller concurrency control, circuit breaking, command timeouts, artifact retention, or download policy. -- Supporting on-premises SQL Server or self-hosted deployments in the baseline. -- Versioning the result contracts independently of the FHIR database schema. - -## Platform and disclosure boundary - -### Azure SQL Database - -The baseline targets Azure SQL Database and uses only Query Store catalog columns guaranteed across the supported Azure SQL deployment fleet at implementation time. Optional columns that may be rolling out regionally must not be referenced until they are universally available. - -### Query Store wait-stat observability - -When `sys.database_query_store_options.wait_stats_capture_mode_desc` is `ON`, the slow-query procedure reads `sys.query_store_wait_stats` directly. `WaitStatsStatus` is `Available` for `ON`, `Disabled` for `OFF`, and `Unavailable` for any other value. This lets authorized direct SQL and future Geneva callers retrieve the complete slow-query diagnostic payload—runtime metrics, waits, query text, and plan IDs—from one interface. Azure SQL may also export `QueryStoreWaitStatistics` to Log Analytics, but that stream does not provide the query text or Showplan XML returned by these procedures. - -`DatabaseWaitStatistics`, when enabled, remains separate database-level telemetry. It is not plan-level data and is not a substitute for the Query Store wait statistics returned with each slow-query plan. +## Design -### Query Store plans are estimated plans - -`sys.query_store_plan.query_plan` contains the compile-time Showplan, equivalent to `SET SHOWPLAN_XML ON`. Query Store combines this plan with aggregated runtime statistics; it does not retain an actual plan for every execution. - -The baseline reserves the future procedure name: +A **watchdog** — the repository's existing leased background-worker pattern — periodically reads Query Store and statistics metadata and **pushes** the results out as metrics notifications. It runs inside the FHIR server process on the server's **existing** SQL identity. ```text -dbo.GetLastActualQueryPlanDiagnostics +FHIR server instance (lease holder) + └── QueryStoreDiagnosticsWatchdog every PeriodSec, default 3600s + ├── sys.database_query_store_options state check, read-only + ├── sys.query_store_* slow plans + query text + ├── sys.query_store_wait_stats best effort + ├── sys.stats / sys.dm_db_stats_properties + └── QueryPlanSanitizer (C#) strips parameter values + │ + └── IMediator.PublishAsync(IMetricsNotification) + └── host-supplied handler (PaaS -> Geneva / Log Analytics) ``` -This is documentation only. No stub procedure, shared output contract, permission grant, Query Store text execution, `LAST_QUERY_PLAN_STATS` enablement, or plan-cache lookup is included. - -## Implementation simplifications - -- Plan-type and Parameter Sensitive Plan dispatcher/query-variant metadata are intentionally not read. Those Azure SQL catalog fields are not stable across the supported deployment fleet, so both Query Store procedures omit them rather than returning speculative NULL/status fields or attempting version-specific fallback logic. -- `GetStatisticsHealth` reports database-level `sys.dm_db_stats_properties` metadata for each statistics object. It does not expand incremental statistics into partition-level property rows; unavailable properties remain `NULL` and are explicitly marked `PropertiesUnavailable`. Its table-name input is materialized as `nvarchar(128)`, rather than the type-equivalent `sysname` alias, because the existing schema C# model generator interprets `sysname` as a table-valued parameter. - -### Accepted query and plan content - -Query text, statement text, scalar expressions, non-parameter constants, object names, missing-index recommendations, warnings, memory grants, and optimizer statistics usage are permitted diagnostic output. - -This intentionally accepts that ad hoc or non-parameterized query text and plan constants may contain literal values. The protected content is parameter-value metadata contained in Showplan `ParameterList` elements, including compiled and runtime parameter values. - -Statistics histogram values remain excluded because `range_high_key` contains actual indexed-column values. - -## Repository ownership - -### OSS `fhir-server` - -The OSS repository owns the persistent database contract: - -- stored procedure definitions; -- `FhirDiagnosticsReader`; -- individual procedure grants; -- schema version and migration scripts; -- SQL aggregation, sanitization, permission, and compatibility tests; and -- the canonical SQL interface documentation. - -These objects must be part of the normal `Microsoft.Health.Fhir.SqlServer` schema artifacts and applied by `Microsoft.Health.Fhir.SchemaManager`. A database at the corresponding schema version must not depend on a separate PaaS rollout to acquire them. - -Although the supported operational scenario is FHIR PaaS on Azure SQL Database, the OSS migration must remain safe for databases that consume the OSS SQL schema. PaaS-specific identities, storage accounts, APIs, and rollout mechanisms must not be embedded in the OSS procedures. - -### `fhir-paas` - -The PaaS repository owns the operational integration: - -- selecting the initial caller surface, such as direct support tooling, Script Runner, Geneva, or a PaaS administrative operation; -- mapping an approved managed identity or support principal to `FhirDiagnosticsReader`; -- caller authentication and authorization; -- operation-level concurrency control, circuit breaking, and command timeout; -- invoking the OSS stored procedures without caller-supplied SQL; -- formatting, transporting, retaining, and auditing downloaded result artifacts; and -- coordinating deployment after the required OSS package/schema version is available. +The critical property is that **nothing new connects inbound to the database**. The work is done by the service that already holds a connection, so the feature introduces no new authentication path, no new database principal, and no new grant. -PaaS must consume the procedures through the OSS `Microsoft.Health.Fhir.SqlServer` and `Microsoft.Health.Fhir.SchemaManager` packages. It must not maintain a second PaaS-only schema version or duplicate production `CREATE OR ALTER PROCEDURE` definitions. +### Why a watchdog -The PaaS Script Runner may be used for a temporary read-only prototype or to invoke the deployed stored procedures. It must not be the production installation mechanism for these persistent objects. Otherwise a database could report the current OSS schema version while silently lacking the diagnostic procedures or role. +`Watchdog` already provides everything this feature needs, and every one of these behaviours would otherwise have to be reinvented: -### Rollout dependency +- **Single-runner election.** `WatchdogLease` ensures exactly one instance in a multi-instance deployment performs the work, so an eight-instance service does not issue eight concurrent Query Store scans. +- **Runtime-tunable period.** `PeriodSec` is seeded into `dbo.Parameters` on first run and re-read from there, so the interval can be changed on a live database without a redeploy. +- **An established runtime override.** `DefragWatchdog` already uses a `{Name}.IsEnabled` row in `dbo.Parameters` as an operational switch. This feature reuses that idiom. +- **A precedent for emitting SQL telemetry.** `GeoReplicationLagWatchdog` reads a SQL view on a timer and publishes an `IMetricsNotification`; the host binds a handler that forwards it. This feature is the same shape. -The rollout order is: +### Enablement -1. merge and release the OSS schema change; -2. update `fhir-paas` to consume the OSS package containing that schema version; -3. allow the existing PaaS schema manager flow to apply the migration; -4. provision approved role membership; and -5. enable the PaaS invocation and artifact-handling workflow. +The feature is **off by default** and is gated by two independent switches. Both must be true before any Query Store read occurs. -## Stored procedures +| Gate | Location | Purpose | +| --- | --- | --- | +| `FhirServer:Watchdog:QueryStoreDiagnostics:Enabled` | Host configuration | Deployment-time gate. When false the watchdog is never started by `WatchdogsBackgroundService`. | +| `QueryStoreDiagnosticsWatchdog.IsEnabled` | `dbo.Parameters` | Runtime gate. Lets a single account be switched on or off against a live database without a redeploy or restart. | -All procedures: +This two-gate arrangement is what makes the feature safe to ship dark: the configuration gate keeps it off for the fleet, and the `dbo.Parameters` gate lets an investigation be turned on for one affected account and turned off again afterwards. -- use `WITH EXECUTE AS 'dbo'`; -- use `SET NOCOUNT ON`; -- use no dynamic SQL; -- create no explicit transaction; -- do not change session isolation level, `LOCK_TIMEOUT`, or `XACT_ABORT`; -- return exactly one result set; -- use repository-standard `THROW` errors for invalid calls and unavailable prerequisites; and -- write Start, End, and Error events through `dbo.LogEvent`. +### Configuration -### 1. `dbo.GetQueryStoreSlowQueries` +`WatchdogConfiguration.QueryStoreDiagnostics`, bound from `FhirServer:Watchdog:QueryStoreDiagnostics`. Note that `Watchdog` is a sibling of `Operations` under `FhirServer`, not nested inside it: -Returns one row per `query_id + plan_id` for regular executions in Query Store runtime intervals overlapping the requested time range. +| Setting | Default | Meaning | +| --- | --- | --- | +| `Enabled` | `false` | Deployment-time gate described above. | +| `PeriodSec` | `3600` | Interval between collections. Seeds `dbo.Parameters`; the live value is read from there. Also used as the Query Store lookback window. | +| `SlowQueryCount` | `10` | Number of slow plans to report per tick. | +| `MinDurationMilliseconds` | `1000` | Minimum weighted average duration for a plan to be reported. | +| `IncludeQueryPlans` | `true` | Whether sanitized Showplan XML is emitted. | +| `IncludeStatisticsHealth` | `true` | Whether statistics metadata is emitted. | +| `StatisticsHealthCount` | `20` | Number of statistics rows to report per tick. | -#### Inputs +The lookback window is the live `PeriodSec` clamped to `[60, 86400]` seconds, so the collection window tracks the collection interval and a misconfigured value cannot request an unbounded scan. -| Parameter | Behavior | -|---|---| -| `@StartTime datetimeoffset = NULL` | Defaults to one hour before the resolved `@EndTime`. | -| `@EndTime datetimeoffset = NULL` | Defaults to `SYSUTCDATETIME()`. | -| `@Top int = 20` | Must be between 1 and 100. | -| `@Offset int = 0` | Must be between 0 and 10,000. `@Offset = 10000` may still return up to 100 rows. | -| `@OrderBy varchar(32) = 'TotalDuration'` | Case-insensitive allowlist described below. | -| `@MinExecutions bigint = 1` | Must be a positive `bigint`. | -| `@QueryTextContains nvarchar(256) = NULL` | Optional literal substring filter. After trimming, it must contain 3-256 characters. | +## Emitted contracts -`@StartTime` and `@EndTime` accept explicit offsets and are normalized to UTC. The start must precede the end, and the requested range must not exceed 24 hours. +Three notification types implement `IMetricsNotification`, each reporting `FhirOperation` `query-store-diagnostics` and `ResourceType` `System`. Hosts bind handlers to route them; the OSS repository does not prescribe a sink. -`@QueryTextContains`: +### `SlowQueryNotification` -- is the only query-content filter; -- is matched under the database collation; -- may contain any caller-supplied text; -- is treated as a literal substring, not a caller-defined `LIKE` pattern; -- escapes `~`, `%`, `_`, and `[` and uses an explicit `ESCAPE N'~'` clause; -- rejects whitespace-only values; and -- is never written to `dbo.LogEvent`. +One per slow plan per tick. Carries `QueryId`, `PlanId`, execution count, total/average/maximum duration, total/average CPU, total/average logical reads, total/average wait time, top wait category, the Query Store query text, and the collection window bounds. -The `@OrderBy` allowlist is: +`QueryId` and `PlanId` are the join keys back into Query Store, and into the `QueryStoreRuntimeStatistics` stream already exported to Log Analytics, so a responder can correlate an emitted slow query with the existing diagnostic-settings telemetry. -- `TotalDuration` -- `AverageDuration` -- `MaximumDuration` -- `TotalCpu` -- `AverageCpu` -- `LogicalReads` -- `Executions` -- `TotalWait` +### `QueryPlanNotification` -Unknown order values fail explicitly. Ordering uses a static `CASE` expression rather than dynamic SQL. Diagnostic metrics sort descending, NULL wait totals sort last, and `query_id ASC, plan_id ASC` are deterministic tie-breakers. +One per reported plan per tick when `IncludeQueryPlans` is set. Carries `QueryId`, `PlanId`, the sanitized Showplan XML, a truncation flag, the raw and sanitized plan lengths, and a sanitization status. -There is no execution-type input in the baseline. Runtime and wait metrics include regular executions only. The implementation should contain a focused comment identifying where execution-type support could be added later. +Notifications are emitted for unsuccessful sanitization as well, with null XML and a status of `PlanXmlUnavailable`, `InvalidXml`, or `VerificationFailed`, so a sanitization failure is observable rather than silent. -#### Time-window semantics +### `StatisticsHealthNotification` -Query Store runtime rows are interval aggregates. The procedure includes every interval that overlaps the half-open requested range `[StartTime, EndTime)`. Edge intervals may therefore include executions immediately outside the requested timestamps. The result does not repeat the resolved request window or interval boundaries. - -#### Runtime aggregation - -Query Store can expose multiple in-memory and persisted rows for the active interval. Runtime data must first collapse rows by: - -```text -plan_id + execution_type + runtime_stats_interval_id -``` - -It is then rolled up by `query_id + plan_id`. - -Weighted totals and averages use `decimal(38,4)` intermediates: - -```text -total duration = SUM(avg_duration * count_executions) -average duration = total duration / SUM(count_executions) -``` +One per statistics object per tick when `IncludeStatisticsHealth` is set. Carries schema, table, and statistics name, last-updated timestamp, rows, rows sampled, modification counter, modification percentage, and the auto-created / user-created / from-index / filtered flags. -The same weighting applies to CPU, reads, writes, and row count. Totals are returned as `decimal(38,0)` and averages as `decimal(38,4)`. +### Field size -Minimum values use the minimum of interval minima. Maximum values use the maximum of interval maxima. Last values come from the row with the latest execution time, using runtime interval ID and runtime-statistics row ID descending as deterministic tie-breakers. +Query text and sanitized plan XML are capped at **32 KB** per field, matching the field limit of the downstream telemetry pipeline. Values exceeding the cap are truncated and flagged (`QueryTextTruncated`, `QueryPlanTruncated`). -Query-level compile count and last compile time are repeated on each plan row and must be clearly named as query-level metadata. +The pre-truncation length is reported alongside each flag so the loss is quantifiable rather than merely visible: `QueryTextLength` for query text, and `SanitizedQueryPlanLength` for plan XML. Plans additionally report `OriginalQueryPlanLength`, the raw length as read from Query Store — the two plan lengths differ by whatever sanitization removed, so reporting only the raw length would overstate how much truncation itself discarded. -Plans with fewer than `@MinExecutions` regular executions in the selected window are excluded. The diagnostic procedures' own Query Store entries are also excluded. All other object-bound and ad hoc Query Store entries are eligible. +Truncation is applied **after** sanitization and verification, never before, so a truncated plan can never be a partially sanitized one. -#### Wait statistics +## Behaviour -Wait statistics are aggregated for the same regular-execution population and overlapping intervals as runtime metrics. +### Query Store state -Each result row contains: +The watchdog reads `sys.database_query_store_options` and proceeds only when `actual_state_desc` is `READ_WRITE`. Any other state, or no row at all, is logged as a warning naming the state and `readonly_reason`, and the tick is skipped. -- `TotalWaitMilliseconds` -- `AverageWaitMilliseconds` -- `WaitStatsStatus` -- `WaitStatsXml` +**The watchdog never issues `ALTER DATABASE`.** Turning Query Store on is a database-scoped configuration change with its own permission and blast-radius considerations, and it is on by default in Azure SQL Database. Enabling it stays an explicit operator action. -`WaitStatsXml` contains one element per wait category, ordered by total wait descending, with: - -- category name; -- total wait milliseconds; -- average wait milliseconds; and -- maximum wait milliseconds. - -Zero-wait categories are omitted. When wait capture is available but a plan has no waits, the value is an empty typed root such as ``. When wait capture is disabled or unavailable, `WaitStatsXml` and scalar wait metrics are NULL and `WaitStatsStatus` explains the condition. Other runtime results still return. - -If `@OrderBy = 'TotalWait'` while wait capture is disabled or unavailable, rows still return. NULL wait totals sort last. - -#### Output - -The single result set includes: - -- `query_id` -- `plan_id` -- `query_hash` -- `query_plan_hash` -- full `query_sql_text` -- `object_id` -- `object_name`, without a separate schema-name column -- regular execution count -- total, average, minimum, maximum, and last duration in explicitly named microsecond columns -- total and average CPU in explicitly named microsecond columns -- total and average logical reads -- total and average physical reads -- total and average logical writes -- average and maximum row count -- first and last execution time in UTC -- query-level compile count and last compile time -- forced-plan state and available force-failure metadata -- other universally available diagnostic plan metadata; plan-type, dispatcher, and query-variant metadata are omitted -- total and average wait milliseconds -- `WaitStatsStatus` -- `WaitStatsXml` - -Physical reads and writes are output metrics but are not ordering options. Query context/handle metadata and execution type are omitted. - -Readable Query Store `READ_WRITE` and `READ_ONLY` states return available data. The actual state and read-only reason are logged. `OFF`, `ERROR`, or otherwise unreadable states fail explicitly. A readable store with no qualifying rows returns no rows. - -### 2. `dbo.GetQueryStorePlanDiagnostics` - -Accepts one required `@PlanId bigint` and returns one row containing Query Store metadata, full query text, and a parameter-redacted Showplan. - -An unknown or evicted plan ID fails explicitly with a stable "plan not found or no longer retained" error. The procedure does not fall back to another plan or constrain the plan by a time range. - -#### Output - -The result includes: - -- `PlanId` -- `QueryId` -- `QueryHash` -- `QueryPlanHash` -- full `QuerySqlText` -- engine and compatibility versions -- compile metadata -- trivial, parallel, forced-plan, and force-failure metadata -- first and last execution metadata when available -- `SanitizationStatus` -- `SanitizationErrorCode` -- `SanitizedShowPlanXml` - -The entire multi-statement Showplan document is preserved. There is no separate allowlisted `PlanDiagnosticsXml`. -Plan-type, dispatcher, and query-variant metadata are unavailable on the baseline catalog and are omitted. - -#### Showplan sanitization - -The raw Query Store plan must never be returned. The procedure: - -1. copies `query_plan` into a local `xml` variable; -2. counts all elements whose local name is `ParameterList`, regardless of namespace; -3. removes every such element in a single XML DML operation; -4. verifies structurally that no `ParameterList` element and no `ParameterCompiledValue` or `ParameterRuntimeValue` attribute remains; -5. serializes the result and performs a case-insensitive textual check for those forbidden names; and -6. returns the XML only when every verification succeeds. - -Unknown Showplan namespaces are processed using the same namespace-agnostic removal and verification. All content other than `ParameterList` elements is preserved, including statement text, non-parameter constants, object/index names, missing-index recommendations, warnings, memory grants, optimizer statistics usage, and plan shape. - -There is no serialized plan-size cap. - -#### Partial availability - -If the plan row exists but `query_plan` is NULL, return the safe metadata with: - -- `SanitizedShowPlanXml = NULL`; -- `SanitizationStatus = 'PlanXmlUnavailable'`; and -- a stable non-sensitive error code. - -If the XML cannot be parsed or redaction verification fails, return safe metadata only with: +### Slow-query aggregation -- `SanitizedShowPlanXml = NULL`; -- `SanitizationStatus = 'InvalidXml'` or `'VerificationFailed'`; and -- a stable non-sensitive error code. +Runtime statistics are aggregated across every Query Store interval overlapping the lookback window, restricted to `execution_type = 0` so that only regular completed executions contribute. Averages are weighted by `count_executions` before being combined across intervals, because Query Store stores per-interval averages and an unweighted mean across intervals of unequal execution counts is wrong. -Detailed parser messages must not be returned because they may echo plan content. These conditions also write an Error audit event. Raw or partially sanitized XML is never returned as a fallback. +Query Store records durations and CPU in **microseconds**; the emitted contract is in milliseconds. -### 3. `dbo.GetStatisticsHealth` +Results are ordered by total duration descending and limited to `SlowQueryCount`, so the reported set is the aggregate-cost hot list rather than the worst single execution. -Returns one row per statistics object for user tables. +### Self-exclusion -#### Inputs +The watchdog's own Query Store reads are excluded by filtering out query text referencing the Query Store catalog views. An earlier iteration attempted to tag the queries with a marker comment; SQL Server does **not** preserve those comments in `query_sql_text`, so text matching on the view names is the pragmatic mechanism. -| Parameter | Behavior | -|---|---| -| `@TableName nvarchar(128) = NULL` | Optional exact table-name filter under the database collation. | -| `@Top int = 20` | Must be between 1 and 100. | -| `@Offset int = 0` | Must be between 0 and 10,000. | -| `@OrderBy varchar(32) = 'ModificationPercent'` | Case-insensitive allowlist described below. | +### Wait statistics -There is no statistics-name filter and no minimum modification count/percentage filter. +Wait statistics are collected by a **separate, best-effort** query and merged in C#. A failure — most commonly `sys.query_store_wait_stats` being unavailable, or wait capture being off — is logged at debug level and leaves the wait fields null. Runtime results are still emitted. -A supplied table name must be nonblank and resolve to exactly one user table. Unknown names fail explicitly. The baseline assumes FHIR operational tables do not span multiple schemas, so table-name input and output omit schema. +Wait capture is retained locally, rather than deferred entirely to the `QueryStoreWaitStatistics` Log Analytics stream, so that a single emitted slow-query record is self-contained: runtime metrics, waits, query text, and plan identity arrive together without a join against a second telemetry source. -Database-wide results exclude temporal history tables. An exact `@TableName` request may explicitly select a temporal history table. +### Statistics health -The `@OrderBy` allowlist is: +Statistics are read from `sys.stats` with an `OUTER APPLY` to `sys.dm_db_stats_properties`, so a statistics object remains visible even when its properties cannot be read. The scan is restricted to user tables and excludes temporal history tables. Rows are ordered by staleness — modification counter over row count — and limited to `StatisticsHealthCount`. -- `ModificationCount` -- `ModificationPercent` -- `LastUpdated` -- `SamplingPercent` -- `Rows` +Modification percentage is left null when the row count is null or zero rather than being reported as zero, so "no data" is distinguishable from "not stale". Percentages above 100 are preserved; they are a legitimate signal that a table has churned more than its cardinality. -Unknown values fail explicitly. Modification count, modification percentage, sampling percentage, and rows sort descending. `LastUpdated` places NULL values first and then sorts oldest first. Table name and statistics name ascending are deterministic tie-breakers. +### Failure containment -#### Sources +`WatchdogsBackgroundService` cancels **every** watchdog if any one of them fails. A diagnostic feature must never be able to take down transaction or cleanup watchdogs, so the collection body contains its own failures: missing views and permission denials are logged and the tick returns rather than propagating. -- `sys.tables` -- `sys.stats` -- `sys.stats_columns` -- `sys.columns` -- `sys.indexes` -- `sys.dm_db_stats_properties` +## Sanitization -The procedure includes index, user-created, and auto-created statistics. Memory-optimized tables are included when compatible metadata is available. Microsoft-shipped and internal tables are excluded. +Showplan sanitization is performed **in C#** by `QueryPlanSanitizer`, not in T-SQL. The previous design did this with 236 lines of XML DML inside a stored procedure; the C# implementation is materially more reliable and easier to test with a fixture corpus. -#### Output +The sanitizer: -The result includes: +1. parses with an `XmlReader` configured with `DtdProcessing.Prohibit` and a null resolver, so a hostile or malformed plan cannot trigger entity resolution; +2. removes every `ParameterList` element and every `ParameterCompiledValue` and `ParameterRuntimeValue` attribute, matching on local name so that Showplan namespace differences between SQL versions cannot cause a miss; +3. re-serializes without formatting; +4. **verifies** that none of those three names survive anywhere in the serialized output, and returns `VerificationFailed` with null XML if any do; and +5. only then truncates to the field cap. -- table name -- statistics name -- statistics ID -- ordered typed XML containing each statistics-column ordinal and name -- auto-created and user-created flags -- incremental, persisted-sample, and no-recompute flags -- filtered-statistics flag and full `filter_definition` -- associated index ID, name, and type description -- disabled and hypothetical index flags -- last update time in UTC -- decimal `HoursSinceLastUpdate` -- row and unfiltered-row counts -- sampled rows -- sampling percentage -- histogram step count, but not histogram contents -- modification counter -- uncapped modification percentage calculated as `modification_counter / rows` -- `StatisticsStatus` +Step 4 is defence in depth: the plan is never emitted on the strength of the removal logic alone. -Sampling and modification percentages are NULL when their denominator is zero or required properties are unavailable. Modification percentages may exceed 100 percent. +### Disclosure boundary -If `sys.dm_db_stats_properties` returns no row, the statistics object remains in the result with property fields NULL and `StatisticsStatus = 'PropertiesUnavailable'`. +Query text, statement text, scalar expressions, non-parameter constants, object names, missing-index recommendations, warnings, memory grants, and optimizer statistics usage are permitted diagnostic output. -Incremental statistics expose only the incremental flag. Partition-level properties are out of scope. +This intentionally accepts that ad hoc or non-parameterized query text and plan constants may contain literal values. The protected content is parameter-value metadata in Showplan `ParameterList` elements, including compiled and runtime values. -The procedure must not call `DBCC SHOW_STATISTICS`, `sys.dm_db_stats_histogram`, or any source that returns histogram keys or density vectors. +Statistics histogram values are never read, because `range_high_key` contains actual indexed-column values. ## Security model -### Database role - -Create the database role: - -```sql -CREATE ROLE FhirDiagnosticsReader; -``` - -Grant the role `EXECUTE` individually on: - -- `dbo.GetQueryStoreSlowQueries` -- `dbo.GetQueryStorePlanDiagnostics` -- `dbo.GetStatisticsHealth` +The watchdog runs on the FHIR server's existing SQL connection and requires no additional database principal, role, or grant. Reading the Query Store catalog views requires `VIEW DATABASE STATE`, which the service identity already holds; where it does not, the permission denial is contained and logged rather than being fatal. -Future diagnostic procedures require an explicit reviewed grant. Do not grant schema-level execution on `dbo`. +Because this is an outbound push from a process that is already trusted with the data, the previous design's audit requirements do not apply. There is no external caller to attribute, and the emitted notifications are themselves the operational record. -The role receives no: - -- `db_datareader`; -- direct `SELECT` on FHIR tables; -- direct Query Store catalog access; -- `VIEW DATABASE STATE`; -- arbitrary command execution; or -- direct `EXECUTE` permission on `dbo.LogEvent`. - -Existing database administrators can use the procedures immediately through their existing privileges. The role is available for future least-privilege callers, with membership controlled independently in each environment. - -The "Reader" name describes the externally observable diagnostic behavior. Internal audit writes do not alter Query Store, statistics, plan cache, FHIR data, or schema. - -### Caller integration - -The initial caller remains undecided. A direct SQL connection, PaaS administrative operation, Geneva action, or other approved tool may invoke the same SQL contract. +## Repository ownership -Caller authentication, authorization, concurrency control, circuit breaking, command timeout, result retention, and download policy are caller responsibilities. Returned query text and sanitized Showplan are treated as operational metadata, but full artifacts must not be written to general logs, metrics dimensions, or `dbo.LogEvent`. +### OSS `fhir-server` -For the managed PaaS service, these caller responsibilities are implemented in `fhir-paas`; they are not added to the OSS schema migration. +- the watchdog, its inline SQL, and the C# sanitizer; +- the configuration class and its defaults; +- the three notification contracts; +- unit tests for sanitization and integration tests against a live SQL Server; and +- this specification. -## Resource controls +Nothing here is PaaS-specific, and no PaaS identity, storage account, or rollout mechanism is embedded in it. A self-hosted deployment can enable the feature and bind its own handler. -SQL enforces: +### `fhir-paas` -- a default one-hour and hard maximum 24-hour slow-query window; -- `@Top <= 100`; -- `@Offset <= 10000`; -- a positive `@MinExecutions`; -- a 3-256-character literal query-text substring; -- one plan per plan-diagnostics call; -- static SQL only; -- regular-execution-only runtime and wait aggregation; and -- exclusion of the diagnostic procedures' own Query Store entries. +- binding notification handlers and routing the emissions to Geneva or Log Analytics; +- setting the configuration gate per environment and ring; +- operating the `dbo.Parameters` runtime override during an investigation; +- retention, access control, and downstream handling of emitted query text and plans; and +- any responder-facing tooling built on top of the emitted stream. -Limits are hard-coded in the procedures. There is no `dbo.Parameters` kill switch, SQL concurrency gate, plan-size cap, total-count query, continuation token, or `HasMoreRows` result. +### Rollout -## Audit and observability +1. Merge the OSS change. The feature ships disabled. +2. Bind a handler and configure routing in `fhir-paas`. +3. Enable the configuration gate in a test ring and confirm emission volume and field sizes. +4. Enable per account through the `dbo.Parameters` override during investigations. -Every procedure follows the existing `dbo.LogEvent` pattern and writes Start, End, and Error events, including successful calls. +Because there is **no schema change**, there is no migration ordering dependency and no package/schema-version synchronization step. This is the single largest operational simplification relative to the rejected design. -Audit records include: +## Simplifications and deferred work -- `ORIGINAL_LOGIN()`; -- effective database principal from `USER_NAME()`; -- procedure name; -- bounded request metadata; -- elapsed milliseconds; -- returned row count; and -- sanitized XML size for successful plan retrieval. +Recorded deliberately; each is a candidate for a follow-up. -Slow-query audit metadata includes resolved UTC window, ordering mode, `@Top`, `@Offset`, `@MinExecutions`, Query Store state/read-only reason, wait-statistics capture mode, and query-text filter presence/length, but never the filter text or wait payload. +- **No schema version bump.** The SQL is inline in the watchdog rather than in versioned stored procedures. This removes a schema version, a 944-line migration diff, a database role, migration-sync risk, and the unresolved `dbo.LogEvent` audit-registration question. The cost is that the SQL is not independently hotfixable through a schema migration, and is reviewed as C# rather than as `.sql`. The SQL is kept in clearly formatted, commented `const` blocks to preserve readability. +- **Plans are re-emitted every tick.** There is no cross-tick deduplication by `plan_id`, so a persistently slow plan is emitted repeatedly. Accepted for v1; the duplication is bounded by `SlowQueryCount` and the tick interval, and suppression can be added once real emission volume is known. +- **Slow-query selection is total-duration only.** There is no configuration for selecting by CPU, reads, waits, or regression against a baseline. `MinDurationMilliseconds` and `SlowQueryCount` are the only tuning knobs. Richer selection is the expected first enhancement. +- **Truncation is a hard cut.** A plan exceeding 32 KB is truncated to invalid XML and flagged. It is not chunked, compressed, or externalized to blob storage. Compression would likely bring most large plans under the cap and is the obvious next step if truncation proves common. +- **No actual execution plans.** `sys.query_store_plan.query_plan` is the compile-time Showplan, equivalent to `SET SHOWPLAN_XML ON`. Actual-plan capture via `LAST_QUERY_PLAN_STATS` remains future work. +- **Plan-type and Parameter Sensitive Plan variant metadata are not read**, because those catalog fields are not stable across the supported Azure SQL fleet. +- **Statistics are reported at database level.** Incremental statistics are not expanded into partition-level rows. -Plan audit metadata includes `plan_id`, sanitization status, stable error code, and result size, but never query text or XML. +## Rejected alternative: caller-invoked stored procedures -Statistics audit metadata includes the exact validated table name when supplied, ordering mode, `@Top`, and `@Offset`. +The original design exposed `dbo.GetQueryStoreSlowQueries`, `dbo.GetQueryStorePlanDiagnostics`, and `dbo.GetStatisticsHealth` behind a `FhirDiagnosticsReader` execute-only role, to be called by an external operational caller. -Audit records must not contain: +It was rejected because it requires an **outside principal to connect to the FHIR database and execute procedures**, which is an entirely new inbound permission model for this service. That model would have to be provisioned, granted, audited, rotated, and defended in every environment, for a diagnostic feature. The watchdog design achieves the same investigative outcome using a trust relationship that already exists. -- query text; -- `@QueryTextContains`; -- Showplan XML; -- parameter values; -- histogram values; or -- every query/plan ID returned by a page. +Secondary benefits of the change: -If Start, End, or Error logging fails, the diagnostic call fails. Callers do not receive direct permission to invoke `dbo.LogEvent`. +- plan sanitization moves from T-SQL to C#, where it is more reliable and far easier to test; +- the persistent SQL surface, the database role, and the schema version all disappear; and +- results are pushed into telemetry continuously rather than requiring someone to be connected and asking at the moment the problem is happening. ## Testing requirements -**Deferred prototype validation:** The prototype has one representative SQL-backed end-to-end path. Exhaustive matrix validation remains future work, including negative validation, fail-closed audit behavior, permissions, a malformed-fixture corpus, and wait-disabled cases. +### Sanitization, unit tested -### Slow-query aggregation +1. Fixtures cover single- and multi-statement plans; compiled values; runtime values; multiple `ParameterList` elements; plans with no parameters; unusual or unknown namespaces; large and deeply nested plans; and malformed XML. +2. Fixtures contain PHI-shaped parameter values. +3. Serialized output contains no `ParameterList`, `ParameterCompiledValue`, or `ParameterRuntimeValue`. +4. Statement text, non-parameter constants, missing-index recommendations, and warnings survive unchanged. +5. Null, malformed, and verification-failing input yields the correct status and null XML. +6. Raw or partially sanitized XML is never returned. +7. Truncation sets the flag and reports the original length, and only ever occurs after successful verification. + +### Collection, integration tested against live SQL -1. Duplicate active-interval in-memory/persisted rows are collapsed before rollup. -2. Weighted totals and averages use the agreed decimal precision. -3. Minimum, maximum, and deterministic last-value calculations are correct. -4. Only regular executions contribute to runtime and wait metrics. -5. Overlapping Query Store interval semantics are verified at both window boundaries. -6. Time, row, offset, minimum-execution, query-text, and order allowlists cannot be bypassed. -7. Literal query-text matching correctly escapes `~`, `%`, `_`, and `[`. -8. Query Store `READ_WRITE` and readable `READ_ONLY` states return data. -9. Query Store `OFF`, `ERROR`, and unreadable states fail with actionable errors. -10. Wait capture available, disabled, unavailable, empty, and `TotalWait` ordering cases are covered. -11. Diagnostic procedures exclude their own Query Store entries. - -### Showplan sanitization - -1. Fixtures include single- and multi-statement plans. -2. Fixtures include compiled values, runtime values, multiple `ParameterList` elements, plans without parameters, unusual namespaces/extensions, PSP/variant plans when available, large/deep plans, and malformed XML. -3. Fixtures contain PHI-shaped parameter values. -4. Serialized output contains no `ParameterList`, `ParameterCompiledValue`, or `ParameterRuntimeValue`. -5. Statement text, non-parameter constants, missing-index recommendations, warnings, and other non-parameter content remain unchanged. -6. Unknown namespaces sanitize successfully when verification passes. -7. NULL, malformed, and verification-failing XML returns metadata only with the correct stable status/code. -8. Raw or partially sanitized XML is never returned. -9. Representative sanitized plans are manually verified to open in the SSMS graphical plan viewer. - -### Statistics - -1. All statistics types are returned with correct ordered column XML. -2. Filter definitions, index metadata, disabled/hypothetical flags, and missing-property status are correct. -3. Sampling/modification percentage zero-denominator behavior is correct. -4. Modification percentages above 100 percent are preserved. -5. Database-wide temporal-history exclusion and explicit history-table inclusion are covered. -6. Histogram keys and density vectors never appear. - -### Permissions and integration - -1. A principal with `FhirDiagnosticsReader` can execute all three procedures. -2. Procedures use `EXECUTE AS 'dbo'` and capture both original and effective identities. -3. Start, End, and Error audit behavior is verified, including fail-closed logging failures. -4. Procedures remain read-only except for required audit events. -5. Deterministic fixture tests are supplemented by Azure SQL integration tests for live catalog compatibility. -6. OSS tests verify the persistent SQL contract without depending on PaaS assemblies or infrastructure. -7. PaaS tests verify package/schema-version synchronization, role provisioning, stored-procedure invocation, and artifact handling without duplicating the SQL implementation. - -## Schema and rollout - -### OSS schema change - -- Add all three procedures under `src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs`. -- Add an idempotent role and individual permission migration. -- Introduce all three procedures and the role in one database schema version and migration diff. -- Include the objects in the generated full schema and packaged SchemaManager resources. -- Keep the migration additive and compatible with the previous application release. -- Use normal repository code review and automated/manual validation. No separate security-review gate is required. -- Keep actual-plan diagnostics as a documented future feature only. - -### PaaS integration change - -- Update the OSS FHIR package versions and synchronized target schema version through the existing `fhir-paas` dependency flow. -- Do not copy the stored procedure or role DDL into PaaS Script Runner scripts. -- Add role membership only for the approved operational identity. -- Implement the selected caller, result transport, artifact storage, and operational authorization in `fhir-paas`. -- Deploy schema/package consumption before enabling the caller. -- Control caller rollout and role membership independently in each environment. +1. With Query Store enabled and a deliberately slow query executed, a `SlowQueryNotification` is emitted carrying a matching `QueryId`/`PlanId`. +2. A `QueryPlanNotification` is emitted for that plan with status `Sanitized`. +3. `StatisticsHealthNotification` rows are emitted for user tables. +4. The watchdog performs no work when either gate is off. +5. A non-`READ_WRITE` Query Store state is handled without error and without emission. +6. Wait-statistic unavailability degrades to null wait fields while runtime results are still emitted. +7. The watchdog does not report its own Query Store queries. ## References @@ -555,7 +256,5 @@ If Start, End, or Error logging fails, the diagnostic call fails. Callers do not - [`sys.query_store_query`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-query-transact-sql) - [`sys.query_store_query_text`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-query-store-query-text-transact-sql) - [`sys.database_query_store_options`](https://learn.microsoft.com/sql/relational-databases/system-catalog-views/sys-database-query-store-options-transact-sql) -- [`sys.dm_exec_query_plan_stats`](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-views/sys-dm-exec-query-plan-stats-transact-sql) - [`sys.dm_db_stats_properties`](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-views/sys-dm-db-stats-properties-transact-sql) -- [`sys.dm_db_stats_histogram`](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-views/sys-dm-db-stats-histogram-transact-sql) - [Showplan XML schemas](https://schemas.microsoft.com/sqlserver/2004/07/showplan/) diff --git a/nuget.config b/nuget.config index f7c1b75416..f40187090d 100644 --- a/nuget.config +++ b/nuget.config @@ -1,10 +1,9 @@ - + - @@ -12,11 +11,11 @@ + + + - - - diff --git a/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs new file mode 100644 index 0000000000..e03ba1efef --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Configs +{ + /// + /// Configuration settings for the Query Store diagnostics watchdog. + /// + public class QueryStoreDiagnosticsConfiguration + { + /// + /// Gets or sets a value indicating whether the Query Store diagnostics watchdog can run. + /// The database runtime override must also be enabled. + /// + public bool Enabled { get; set; } = false; + + /// + /// Gets or sets the interval, in seconds, between diagnostics collections. + /// + public double PeriodSec { get; set; } = 3600; + + /// + /// Gets or sets the maximum number of slow query plans reported per collection. + /// + public int SlowQueryCount { get; set; } = 10; + + /// + /// Gets or sets the minimum weighted average plan duration, in milliseconds, to report. + /// + public int MinDurationMilliseconds { get; set; } = 1000; + + /// + /// Gets or sets a value indicating whether sanitized query plans are reported. + /// + public bool IncludeQueryPlans { get; set; } = true; + + /// + /// Gets or sets a value indicating whether table statistics health is reported. + /// + public bool IncludeStatisticsHealth { get; set; } = true; + + /// + /// Gets or sets the maximum number of table statistics rows reported per collection. + /// + public int StatisticsHealthCount { get; set; } = 20; + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Configs/WatchdogConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/WatchdogConfiguration.cs index 8581a1404e..928eaea12b 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/WatchdogConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/WatchdogConfiguration.cs @@ -14,5 +14,10 @@ public class WatchdogConfiguration /// Gets the expired resource cleanup configuration. /// public ExpiredResourceConfiguration ExpiredResource { get; } = new ExpiredResourceConfiguration(); + + /// + /// Gets the Query Store diagnostics watchdog configuration. + /// + public QueryStoreDiagnosticsConfiguration QueryStoreDiagnostics { get; } = new QueryStoreDiagnosticsConfiguration(); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Metrics/QueryPlanNotification.cs b/src/Microsoft.Health.Fhir.Core/Features/Metrics/QueryPlanNotification.cs new file mode 100644 index 0000000000..8c107a35ef --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Metrics/QueryPlanNotification.cs @@ -0,0 +1,67 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Fhir.Core.Features.Metrics +{ + /// + /// Contains a sanitized Query Store execution plan. + /// + public class QueryPlanNotification : IMetricsNotification + { + /// + /// Gets or sets the Query Store query identifier. + /// + public long QueryId { get; set; } + + /// + /// Gets or sets the Query Store plan identifier. + /// + public long PlanId { get; set; } + + /// + /// Gets or sets the sanitized query plan XML, limited to the diagnostics field-length cap. + /// + public string SanitizedQueryPlan { get; set; } + + /// + /// Gets or sets a value indicating whether was truncated. + /// + public bool QueryPlanTruncated { get; set; } + + /// + /// Gets or sets the character length of the raw query plan XML as read from Query Store. + /// + public int OriginalQueryPlanLength { get; set; } + + /// + /// Gets or sets the character length of the sanitized query plan XML before truncation. + /// Compare this against the field cap to see how much lost when + /// is set. Zero when sanitization did not produce a document. + /// + public int SanitizedQueryPlanLength { get; set; } + + /// + /// Gets or sets the outcome of query plan sanitization. + /// + public string SanitizationStatus { get; set; } + + /// + /// Gets or sets the timestamp when the notification was created. + /// + public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow; + + /// + /// Gets the FHIR operation associated with this notification. + /// + public string FhirOperation => "query-store-diagnostics"; + + /// + /// Gets the resource type associated with this notification. + /// + public string ResourceType => "System"; + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs b/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs new file mode 100644 index 0000000000..d254f79cfe --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs @@ -0,0 +1,121 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Fhir.Core.Features.Metrics +{ + /// + /// Contains aggregated Query Store metrics for a slow query plan. + /// + public class SlowQueryNotification : IMetricsNotification + { + /// + /// Gets or sets the Query Store query identifier. + /// + public long QueryId { get; set; } + + /// + /// Gets or sets the Query Store plan identifier. + /// + public long PlanId { get; set; } + + /// + /// Gets or sets the number of regular completed executions in the reporting interval. + /// + public long ExecutionCount { get; set; } + + /// + /// Gets or sets the total execution duration, in milliseconds. + /// + public double TotalDurationMilliseconds { get; set; } + + /// + /// Gets or sets the weighted average execution duration, in milliseconds. + /// + public double AverageDurationMilliseconds { get; set; } + + /// + /// Gets or sets the maximum execution duration, in milliseconds. + /// + public double MaxDurationMilliseconds { get; set; } + + /// + /// Gets or sets the total CPU time, in milliseconds. + /// + public double TotalCpuMilliseconds { get; set; } + + /// + /// Gets or sets the weighted average CPU time, in milliseconds. + /// + public double AverageCpuMilliseconds { get; set; } + + /// + /// Gets or sets the total logical reads. + /// + public double TotalLogicalReads { get; set; } + + /// + /// Gets or sets the weighted average logical reads. + /// + public double AverageLogicalReads { get; set; } + + /// + /// Gets or sets the total observed wait time, in milliseconds, when Query Store wait statistics are available. + /// + public double? TotalWaitMilliseconds { get; set; } + + /// + /// Gets or sets the average observed wait time per execution, in milliseconds, when Query Store wait statistics are available. + /// + public double? AverageWaitMilliseconds { get; set; } + + /// + /// Gets or sets the wait category with the greatest observed wait time, when Query Store wait statistics are available. + /// + public string TopWaitCategory { get; set; } + + /// + /// Gets or sets the query text, limited to the diagnostics field-length cap. + /// + public string QueryText { get; set; } + + /// + /// Gets or sets a value indicating whether was truncated. + /// + public bool QueryTextTruncated { get; set; } + + /// + /// Gets or sets the character length of the query text before truncation, so the amount lost is + /// visible when is set. + /// + public int QueryTextLength { get; set; } + + /// + /// Gets or sets the start of the Query Store reporting interval. + /// + public DateTimeOffset IntervalStart { get; set; } + + /// + /// Gets or sets the end of the Query Store reporting interval. + /// + public DateTimeOffset IntervalEnd { get; set; } + + /// + /// Gets or sets the timestamp when the notification was created. + /// + public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow; + + /// + /// Gets the FHIR operation associated with this notification. + /// + public string FhirOperation => "query-store-diagnostics"; + + /// + /// Gets the resource type associated with this notification. + /// + public string ResourceType => "System"; + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Metrics/StatisticsHealthNotification.cs b/src/Microsoft.Health.Fhir.Core/Features/Metrics/StatisticsHealthNotification.cs new file mode 100644 index 0000000000..fd9a450bf5 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Metrics/StatisticsHealthNotification.cs @@ -0,0 +1,90 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Fhir.Core.Features.Metrics +{ + /// + /// Contains table statistics health information. + /// + public class StatisticsHealthNotification : IMetricsNotification + { + /// + /// Gets or sets the schema that owns the table. + /// + public string SchemaName { get; set; } + + /// + /// Gets or sets the table name. + /// + public string TableName { get; set; } + + /// + /// Gets or sets the statistics object name. + /// + public string StatisticsName { get; set; } + + /// + /// Gets or sets the timestamp when the statistics were last updated. + /// + public DateTimeOffset? LastUpdated { get; set; } + + /// + /// Gets or sets the number of rows represented by the statistics. + /// + public long? Rows { get; set; } + + /// + /// Gets or sets the number of rows sampled to build the statistics. + /// + public long? RowsSampled { get; set; } + + /// + /// Gets or sets the number of modifications since the statistics were last updated. + /// + public long? ModificationCounter { get; set; } + + /// + /// Gets or sets the percentage of represented rows modified since the statistics were last updated. + /// + public double? ModificationPercent { get; set; } + + /// + /// Gets or sets a value indicating whether SQL Server automatically created the statistics. + /// + public bool IsAutoCreated { get; set; } + + /// + /// Gets or sets a value indicating whether a user created the statistics. + /// + public bool IsUserCreated { get; set; } + + /// + /// Gets or sets a value indicating whether the statistics are associated with an index. + /// + public bool IsFromIndex { get; set; } + + /// + /// Gets or sets a value indicating whether the statistics use a filter. + /// + public bool HasFilter { get; set; } + + /// + /// Gets or sets the timestamp when the notification was created. + /// + public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow; + + /// + /// Gets the FHIR operation associated with this notification. + /// + public string FhirOperation => "query-store-diagnostics"; + + /// + /// Gets the resource type associated with this notification. + /// + public string ResourceType => "System"; + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json index 8ef4b04cc4..890fd74300 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json +++ b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json @@ -159,6 +159,15 @@ "Watchdog": { "ExpiredResource": { "Enabled": false + }, + "QueryStoreDiagnostics": { + "Enabled": false, + "PeriodSec": 3600, + "SlowQueryCount": 10, + "MinDurationMilliseconds": 1000, + "IncludeQueryPlans": true, + "IncludeStatisticsHealth": true, + "StatisticsHealthCount": 20 } } }, diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs new file mode 100644 index 0000000000..b154b76f53 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs @@ -0,0 +1,225 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Xml.Linq; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public class QueryPlanSanitizerTests + { + [Fact] + public void GivenSingleStatementPlanWithPhiShapedParameterValues_WhenSanitized_ThenRemovesParametersAndPreservesDiagnosticContent() + { + // Arrange + const string patientName = "Mikael W"; + const string medicalRecordNumber = "MRN-12345"; + string queryPlan = $@" + + + + + + + + + + + + + + + + + + + + +"; + + // Act + QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, QueryStoreDiagnosticsWatchdog.MaxFieldLength); + + // Assert + Assert.Equal(QueryPlanSanitizer.SanitizedStatus, result.Status); + Assert.NotNull(result.Xml); + Assert.False(result.Truncated); + Assert.Equal(queryPlan.Length, result.OriginalLength); + AssertNoParameterMetadata(result.Xml); + Assert.DoesNotContain(patientName, result.Xml, StringComparison.Ordinal); + Assert.DoesNotContain(medicalRecordNumber, result.Xml, StringComparison.Ordinal); + Assert.Contains("SELECT * FROM dbo.Patient WHERE Status = 1", result.Xml, StringComparison.Ordinal); + Assert.Contains("ConstValue=\"(123)\"", result.Xml, StringComparison.Ordinal); + Assert.Contains("MissingIndexGroup", result.Xml, StringComparison.Ordinal); + Assert.Contains("Warnings", result.Xml, StringComparison.Ordinal); + } + + [Fact] + public void GivenMultiStatementPlanWithMultipleParameterLists_WhenSanitized_ThenRemovesEveryParameterListAndValueAttribute() + { + // Arrange + const string compiledValue = "Alice Smith"; + const string runtimeValue = "MRN-67890"; + string queryPlan = $@" + + + + + + + + + + + + + + + + +"; + + // Act + QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, QueryStoreDiagnosticsWatchdog.MaxFieldLength); + + // Assert + Assert.Equal(QueryPlanSanitizer.SanitizedStatus, result.Status); + Assert.NotNull(result.Xml); + AssertNoParameterMetadata(result.Xml); + Assert.DoesNotContain(compiledValue, result.Xml, StringComparison.Ordinal); + Assert.DoesNotContain(runtimeValue, result.Xml, StringComparison.Ordinal); + Assert.Contains("StatementText=\"SELECT 1\"", result.Xml, StringComparison.Ordinal); + Assert.Contains("StatementText=\"SELECT 2\"", result.Xml, StringComparison.Ordinal); + } + + [Fact] + public void GivenPlansWithUnknownAndNoShowplanNamespaces_WhenSanitized_ThenRemovesParameterMetadataByLocalName() + { + // Arrange + const string unknownNamespacePlan = @""; + const string noNamespacePlan = @""; + + // Act + QueryPlanSanitizationResult unknownNamespaceResult = QueryPlanSanitizer.Sanitize(unknownNamespacePlan, QueryStoreDiagnosticsWatchdog.MaxFieldLength); + QueryPlanSanitizationResult noNamespaceResult = QueryPlanSanitizer.Sanitize(noNamespacePlan, QueryStoreDiagnosticsWatchdog.MaxFieldLength); + + // Assert + Assert.Equal(QueryPlanSanitizer.SanitizedStatus, unknownNamespaceResult.Status); + Assert.Equal(QueryPlanSanitizer.SanitizedStatus, noNamespaceResult.Status); + Assert.NotNull(unknownNamespaceResult.Xml); + Assert.NotNull(noNamespaceResult.Xml); + AssertNoParameterMetadata(unknownNamespaceResult.Xml); + AssertNoParameterMetadata(noNamespaceResult.Xml); + Assert.Contains("SELECT 3", unknownNamespaceResult.Xml, StringComparison.Ordinal); + Assert.Contains("SELECT 4", noNamespaceResult.Xml, StringComparison.Ordinal); + Assert.DoesNotContain("Jane Doe", unknownNamespaceResult.Xml, StringComparison.Ordinal); + Assert.DoesNotContain("MRN-98765", unknownNamespaceResult.Xml, StringComparison.Ordinal); + Assert.DoesNotContain("John Doe", noNamespaceResult.Xml, StringComparison.Ordinal); + Assert.DoesNotContain("MRN-54321", noNamespaceResult.Xml, StringComparison.Ordinal); + } + + [Fact] + public void GivenPlanWithoutParameters_WhenSanitized_ThenPreservesThePlanInSubstance() + { + // Arrange + const string queryPlan = @""; + + // Act + QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, QueryStoreDiagnosticsWatchdog.MaxFieldLength); + + // Assert + Assert.Equal(QueryPlanSanitizer.SanitizedStatus, result.Status); + Assert.NotNull(result.Xml); + Assert.False(result.Truncated); + Assert.Equal(result.Xml.Length, result.SanitizedLength); + Assert.True(XNode.DeepEquals(XDocument.Parse(queryPlan), XDocument.Parse(result.Xml))); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void GivenUnavailablePlanXml_WhenSanitized_ThenReturnsPlanXmlUnavailable(string queryPlan) + { + // Act + QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, QueryStoreDiagnosticsWatchdog.MaxFieldLength); + + // Assert + Assert.Equal(QueryPlanSanitizer.PlanXmlUnavailableStatus, result.Status); + Assert.Null(result.Xml); + Assert.False(result.Truncated); + Assert.Equal(0, result.OriginalLength); + Assert.Equal(0, result.SanitizedLength); + } + + [Fact] + public void GivenMalformedPlanXml_WhenSanitized_ThenReturnsInvalidXmlWithoutThrowing() + { + // Arrange + const string queryPlan = ""; + + // Act + QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, QueryStoreDiagnosticsWatchdog.MaxFieldLength); + + // Assert + Assert.Equal(QueryPlanSanitizer.InvalidXmlStatus, result.Status); + Assert.Null(result.Xml); + Assert.False(result.Truncated); + Assert.Equal(queryPlan.Length, result.OriginalLength); + Assert.Equal(0, result.SanitizedLength); + } + + [Fact] + public void GivenSanitizedPlanExceedingFieldCap_WhenSanitized_ThenReturnsVerifiedTruncatedXml() + { + // Arrange + const int fieldCap = 512; + string queryPlan = $@""; + + // Act + QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, fieldCap); + + // Assert + Assert.Equal(QueryPlanSanitizer.SanitizedStatus, result.Status); + Assert.NotNull(result.Xml); + Assert.True(result.Truncated); + Assert.Equal(queryPlan.Length, result.OriginalLength); + Assert.Equal(fieldCap, result.Xml.Length); + Assert.True(result.SanitizedLength > fieldCap); + Assert.True(result.SanitizedLength <= result.OriginalLength); + AssertNoParameterMetadata(result.Xml); + } + + [Fact] + public void GivenPlanThatFailsVerification_WhenSanitized_ThenNeverReturnsRawXml() + { + // Arrange + const string queryPlan = @""; + + // Act + QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, 1); + + // Assert + Assert.Equal(QueryPlanSanitizer.VerificationFailedStatus, result.Status); + Assert.Null(result.Xml); + Assert.False(result.Truncated); + Assert.Equal(queryPlan.Length, result.OriginalLength); + Assert.True(result.SanitizedLength > 0); + Assert.NotEqual(queryPlan, result.Xml); + } + + private static void AssertNoParameterMetadata(string queryPlan) + { + Assert.DoesNotContain("ParameterList", queryPlan, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("ParameterCompiledValue", queryPlan, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("ParameterRuntimeValue", queryPlan, StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql deleted file mode 100644 index 52b7715346..0000000000 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql +++ /dev/null @@ -1,944 +0,0 @@ -IF NOT EXISTS -( - SELECT 1 - FROM sys.database_principals - WHERE name = N'FhirDiagnosticsReader' - AND type = 'R' -) -BEGIN - IF EXISTS - ( - SELECT 1 - FROM sys.database_principals - WHERE name = N'FhirDiagnosticsReader' - ) - BEGIN - THROW 50100, 'A database principal named FhirDiagnosticsReader already exists but is not a database role.', 1; - END - - CREATE ROLE [FhirDiagnosticsReader]; -END -GO - -CREATE OR ALTER PROCEDURE dbo.GetQueryStoreSlowQueries - @StartTime datetimeoffset(7) = NULL - ,@EndTime datetimeoffset(7) = NULL - ,@Top int = 20 - ,@Offset int = 0 - ,@OrderBy varchar(32) = 'TotalDuration' - ,@MinExecutions bigint = 1 - ,@QueryTextContains nvarchar(256) = NULL -WITH EXECUTE AS 'dbo' -AS -BEGIN - SET NOCOUNT ON; - - DECLARE @ProcedureName varchar(100) = OBJECT_NAME(@@PROCID); - DECLARE @AuditMode varchar(200) = 'QueryStoreSlowQueries'; - DECLARE @AuditStartTime datetime = GETUTCDATE(); - DECLARE @AuditText nvarchar(3500); - DECLARE @RowsReturned bigint; - DECLARE @ResolvedStartTime datetimeoffset(7); - DECLARE @ResolvedEndTime datetimeoffset(7); - DECLARE @OrderByNormalized varchar(32); - DECLARE @QueryTextPattern nvarchar(514); - DECLARE @QueryTextFilterLength int; - DECLARE @QueryStoreState nvarchar(60); - DECLARE @QueryStoreReadOnlyReason bigint; - DECLARE @WaitStatsCaptureMode nvarchar(60); - DECLARE @WaitStatsStatus varchar(32); - DECLARE @SlowQueriesProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStoreSlowQueries'); - DECLARE @PlanDiagnosticsProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStorePlanDiagnostics'); - DECLARE @StatisticsHealthProcedureObjectId int = OBJECT_ID(N'dbo.GetStatisticsHealth'); - - IF @ProcedureName IS NULL - SET @ProcedureName = 'GetQueryStoreSlowQueries'; - - SET @AuditText = CONCAT( - N'OriginalLogin=', ORIGINAL_LOGIN(), - N';EffectivePrincipal=', USER_NAME()); - - BEGIN TRY - -- ------------------------------------------------------------------------ - -- Resolve defaults, capture Query Store state, and build the audit context - -- ------------------------------------------------------------------------ - SET @Top = ISNULL(@Top, 20); - SET @Offset = ISNULL(@Offset, 0); - SET @MinExecutions = ISNULL(@MinExecutions, 1); - SET @OrderBy = ISNULL(@OrderBy, 'TotalDuration'); - SET @ResolvedEndTime = SWITCHOFFSET(ISNULL(@EndTime, TODATETIMEOFFSET(SYSUTCDATETIME(), '+00:00')), '+00:00'); - SET @ResolvedStartTime = SWITCHOFFSET(ISNULL(@StartTime, DATEADD(hour, -1, @ResolvedEndTime)), '+00:00'); - SET @QueryTextContains = NULLIF(LTRIM(RTRIM(@QueryTextContains)), N''); - SET @QueryTextFilterLength = ISNULL(LEN(@QueryTextContains), 0); - - SELECT - @QueryStoreState = actual_state_desc, - @QueryStoreReadOnlyReason = readonly_reason, - @WaitStatsCaptureMode = wait_stats_capture_mode_desc - FROM sys.database_query_store_options; - - SET @WaitStatsStatus = - CASE @WaitStatsCaptureMode - WHEN N'ON' THEN 'Available' - WHEN N'OFF' THEN 'Disabled' - ELSE 'Unavailable' - END; - SET @AuditText = CONCAT( - @AuditText, - N';StartTimeUtc=', CONVERT(nvarchar(33), @ResolvedStartTime, 127), - N';EndTimeUtc=', CONVERT(nvarchar(33), @ResolvedEndTime, 127), - N';OrderBy=', @OrderBy, - N';Top=', @Top, - N';Offset=', @Offset, - N';MinExecutions=', @MinExecutions, - N';QueryTextFilterPresent=', CASE WHEN @QueryTextContains IS NULL THEN N'0' ELSE N'1' END, - N';QueryTextFilterLength=', @QueryTextFilterLength, - N';QueryStoreState=', ISNULL(@QueryStoreState, N'Unknown'), - N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown'), - N';WaitStatsCaptureMode=', ISNULL(@WaitStatsCaptureMode, N'Unknown')); - - EXECUTE dbo.LogEvent - @Process = @ProcedureName, - @Mode = @AuditMode, - @Status = 'Start', - @Text = @AuditText; - - -- ------------------------------------------------------------------------ - -- Validate parameters - -- ------------------------------------------------------------------------ - IF @Top < 1 OR @Top > 100 - THROW 50400, '@Top must be between 1 and 100.', 1; - - IF @Offset < 0 OR @Offset > 10000 - THROW 50401, '@Offset must be between 0 and 10000.', 1; - - IF @MinExecutions < 1 - THROW 50402, '@MinExecutions must be positive.', 1; - - IF @ResolvedStartTime >= @ResolvedEndTime - THROW 50403, '@StartTime must precede @EndTime.', 1; - - IF @ResolvedEndTime > DATEADD(hour, 24, @ResolvedStartTime) - THROW 50404, 'The requested time range must not exceed 24 hours.', 1; - - IF @QueryTextContains IS NOT NULL - AND LEN(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(@QueryTextContains, N' ', N''), NCHAR(9), N''), NCHAR(10), N''), NCHAR(13), N''), NCHAR(160), N'')) = 0 - THROW 50405, '@QueryTextContains must not be whitespace only.', 1; - - IF @QueryTextContains IS NOT NULL - AND (@QueryTextFilterLength < 3 OR @QueryTextFilterLength > 256) - THROW 50406, '@QueryTextContains must contain between 3 and 256 characters after trimming.', 1; - - SET @OrderByNormalized = - CASE - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalDuration' THEN 'TotalDuration' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageDuration' THEN 'AverageDuration' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'MaximumDuration' THEN 'MaximumDuration' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalCpu' THEN 'TotalCpu' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageCpu' THEN 'AverageCpu' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'LogicalReads' THEN 'LogicalReads' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'Executions' THEN 'Executions' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalWait' THEN 'TotalWait' - END; - - IF @OrderByNormalized IS NULL - THROW 50407, '@OrderBy is not supported.', 1; - - -- ------------------------------------------------------------------------ - -- Validate Query Store prerequisite state - -- ------------------------------------------------------------------------ - IF ISNULL(@QueryStoreState, N'') NOT IN (N'READ_WRITE', N'READ_ONLY') - THROW 50408, 'Query Store is not enabled and readable.', 1; - - -- ------------------------------------------------------------------------ - -- Normalize the literal filter into a safe LIKE pattern - -- ------------------------------------------------------------------------ - -- Escape wildcard/escape characters before wrapping so a literal '%', '_', or '[' in the - -- caller-supplied text is matched literally rather than interpreted by LIKE. - SET @QueryTextPattern = - CASE - WHEN @QueryTextContains IS NULL THEN NULL - ELSE N'%' + REPLACE(REPLACE(REPLACE(REPLACE(@QueryTextContains, N'~', N'~~'), N'%', N'~%'), N'_', N'~_'), N'[', N'~[') + N'%' - END; - - -- ------------------------------------------------------------------------ - -- Collapse duplicate runtime-stats rows and compute weighted aggregates - -- ------------------------------------------------------------------------ - -- Active Query Store intervals can expose both a persisted row and an in-memory row for the - -- same plan/interval, so duplicates must be collapsed before aggregating across intervals. - ;WITH RuntimeStatsRows AS - ( - SELECT - rs.plan_id AS PlanId, - rs.execution_type AS ExecutionType, - rs.runtime_stats_interval_id AS RuntimeStatsIntervalId, - rs.runtime_stats_id AS RuntimeStatsId, - rs.count_executions AS RegularExecutionCount, - CONVERT(decimal(38, 4), rs.avg_duration) AS AverageDurationMicroseconds, - CONVERT(decimal(38, 0), rs.min_duration) AS MinimumDurationMicroseconds, - CONVERT(decimal(38, 0), rs.max_duration) AS MaximumDurationMicroseconds, - CONVERT(decimal(38, 0), rs.last_duration) AS LastDurationMicroseconds, - CONVERT(decimal(38, 4), rs.avg_cpu_time) AS AverageCpuMicroseconds, - CONVERT(decimal(38, 4), rs.avg_logical_io_reads) AS AverageLogicalReads, - CONVERT(decimal(38, 4), rs.avg_physical_io_reads) AS AveragePhysicalReads, - CONVERT(decimal(38, 4), rs.avg_logical_io_writes) AS AverageLogicalWrites, - CONVERT(decimal(38, 4), rs.avg_rowcount) AS AverageRowCount, - CONVERT(decimal(38, 0), rs.max_rowcount) AS MaximumRowCount, - SWITCHOFFSET(rs.first_execution_time, '+00:00') AS FirstExecutionTimeUtc, - SWITCHOFFSET(rs.last_execution_time, '+00:00') AS LastExecutionTimeUtc - FROM sys.query_store_runtime_stats AS rs - INNER JOIN sys.query_store_runtime_stats_interval AS rsi - ON rsi.runtime_stats_interval_id = rs.runtime_stats_interval_id - -- The baseline is regular executions only. An explicit execution-type input belongs here if added later. - WHERE rs.execution_type = 0 - -- Query Store interval overlap semantics are inclusive of edge executions, so a run that only - -- partially overlaps the requested [@ResolvedStartTime, @ResolvedEndTime) window is still included. - AND rsi.start_time < @ResolvedEndTime - AND rsi.end_time > @ResolvedStartTime - ), - RankedRuntimeStatsRows AS - ( - SELECT - rs.*, - ROW_NUMBER() OVER - ( - PARTITION BY rs.PlanId, rs.ExecutionType, rs.RuntimeStatsIntervalId - ORDER BY rs.LastExecutionTimeUtc DESC, rs.RuntimeStatsIntervalId DESC, rs.RuntimeStatsId DESC - ) AS LastValueRank - FROM RuntimeStatsRows AS rs - ), - CollapsedRuntimeStats AS - ( - SELECT - rs.PlanId, - rs.ExecutionType, - rs.RuntimeStatsIntervalId, - SUM(rs.RegularExecutionCount) AS RegularExecutionCount, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageDurationMicroseconds * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalDurationMicroseconds, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageCpuMicroseconds * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalCpuMicroseconds, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageLogicalReads * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalLogicalReads, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AveragePhysicalReads * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalPhysicalReads, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageLogicalWrites * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalLogicalWrites, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageRowCount * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalRowCount, - MIN(rs.MinimumDurationMicroseconds) AS MinimumDurationMicroseconds, - MAX(rs.MaximumDurationMicroseconds) AS MaximumDurationMicroseconds, - MAX(rs.MaximumRowCount) AS MaximumRowCount, - MIN(rs.FirstExecutionTimeUtc) AS FirstExecutionTimeUtc, - MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastExecutionTimeUtc END) AS LastExecutionTimeUtc, - MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastDurationMicroseconds END) AS LastDurationMicroseconds, - MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.RuntimeStatsId END) AS LastRuntimeStatsId - FROM RankedRuntimeStatsRows AS rs - GROUP BY - rs.PlanId, - rs.ExecutionType, - rs.RuntimeStatsIntervalId - ), - RankedCollapsedRuntimeStats AS - ( - SELECT - rs.*, - ROW_NUMBER() OVER - ( - PARTITION BY rs.PlanId - ORDER BY rs.LastExecutionTimeUtc DESC, rs.RuntimeStatsIntervalId DESC, rs.LastRuntimeStatsId DESC - ) AS LastValueRank - FROM CollapsedRuntimeStats AS rs - ), - AggregatedRuntimeStats AS - ( - SELECT - rs.PlanId, - SUM(rs.RegularExecutionCount) AS RegularExecutionCount, - CONVERT(decimal(38, 0), SUM(rs.TotalDurationMicroseconds)) AS TotalDurationMicroseconds, - -- Averages are recomputed as execution-count-weighted sums rather than averaged directly, - -- and division uses decimal precision so intervals with unequal execution counts are not skewed. - CONVERT(decimal(38, 4), SUM(rs.TotalDurationMicroseconds) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageDurationMicroseconds, - MIN(rs.MinimumDurationMicroseconds) AS MinimumDurationMicroseconds, - MAX(rs.MaximumDurationMicroseconds) AS MaximumDurationMicroseconds, - MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastDurationMicroseconds END) AS LastDurationMicroseconds, - CONVERT(decimal(38, 0), SUM(rs.TotalCpuMicroseconds)) AS TotalCpuMicroseconds, - CONVERT(decimal(38, 4), SUM(rs.TotalCpuMicroseconds) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageCpuMicroseconds, - CONVERT(decimal(38, 0), SUM(rs.TotalLogicalReads)) AS TotalLogicalReads, - CONVERT(decimal(38, 4), SUM(rs.TotalLogicalReads) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageLogicalReads, - CONVERT(decimal(38, 0), SUM(rs.TotalPhysicalReads)) AS TotalPhysicalReads, - CONVERT(decimal(38, 4), SUM(rs.TotalPhysicalReads) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AveragePhysicalReads, - CONVERT(decimal(38, 0), SUM(rs.TotalLogicalWrites)) AS TotalLogicalWrites, - CONVERT(decimal(38, 4), SUM(rs.TotalLogicalWrites) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageLogicalWrites, - CONVERT(decimal(38, 4), SUM(rs.TotalRowCount) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageRowCount, - MAX(rs.MaximumRowCount) AS MaximumRowCount, - MIN(rs.FirstExecutionTimeUtc) AS FirstExecutionTimeUtc, - MAX(rs.LastExecutionTimeUtc) AS LastExecutionTimeUtc - FROM RankedCollapsedRuntimeStats AS rs - GROUP BY rs.PlanId - HAVING SUM(rs.RegularExecutionCount) >= @MinExecutions - ), - -- ------------------------------------------------------------------------ - -- Aggregate wait-stat rows and handle capture availability - -- ------------------------------------------------------------------------ - WaitStatsRows AS - ( - SELECT - ws.plan_id AS PlanId, - ws.wait_category_desc AS WaitCategoryDescription, - CONVERT(decimal(38, 0), ws.total_query_wait_time_ms) AS TotalWaitMilliseconds, - CONVERT(decimal(38, 0), ws.max_query_wait_time_ms) AS MaximumWaitMilliseconds - FROM sys.query_store_wait_stats AS ws - INNER JOIN sys.query_store_runtime_stats_interval AS rsi - ON rsi.runtime_stats_interval_id = ws.runtime_stats_interval_id - -- When wait capture is disabled or unavailable this CTE is intentionally left empty so runtime - -- rows are still returned with NULL wait columns instead of failing the whole query. - WHERE @WaitStatsStatus = 'Available' - AND ws.execution_type = 0 - AND rsi.start_time < @ResolvedEndTime - AND rsi.end_time > @ResolvedStartTime - ), - WaitCategories AS - ( - SELECT - ws.PlanId, - ws.WaitCategoryDescription, - CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds, - MAX(ws.MaximumWaitMilliseconds) AS MaximumWaitMilliseconds - FROM WaitStatsRows AS ws - GROUP BY - ws.PlanId, - ws.WaitCategoryDescription - ), - AggregatedWaitStats AS - ( - SELECT - ws.PlanId, - CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds - FROM WaitCategories AS ws - GROUP BY ws.PlanId - ), - WaitStatsXml AS - ( - SELECT - wp.PlanId, - ( - SELECT - wc.WaitCategoryDescription AS [@Category], - wc.TotalWaitMilliseconds AS [@TotalWaitMilliseconds], - CONVERT(decimal(38, 4), wc.TotalWaitMilliseconds / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) AS [@AverageWaitMilliseconds], - wc.MaximumWaitMilliseconds AS [@MaximumWaitMilliseconds] - FROM WaitCategories AS wc - WHERE wc.PlanId = wp.PlanId - AND wc.TotalWaitMilliseconds > 0 - ORDER BY - wc.TotalWaitMilliseconds DESC, - wc.WaitCategoryDescription ASC - FOR XML PATH(N'WaitCategory'), ROOT(N'WaitStats'), TYPE - ) AS WaitStatsXml - FROM (SELECT DISTINCT PlanId FROM WaitCategories) AS wp - INNER JOIN AggregatedRuntimeStats AS ars - ON ars.PlanId = wp.PlanId - ) - -- ------------------------------------------------------------------------ - -- Project results with static, deterministic ordering - -- ------------------------------------------------------------------------ - SELECT - q.query_id AS QueryId, - p.plan_id AS PlanId, - q.query_hash AS QueryHash, - p.query_plan_hash AS QueryPlanHash, - qt.query_sql_text AS QuerySqlText, - q.object_id AS ObjectId, - OBJECT_NAME(q.object_id) AS ObjectName, - ars.RegularExecutionCount, - ars.TotalDurationMicroseconds, - ars.AverageDurationMicroseconds, - ars.MinimumDurationMicroseconds, - ars.MaximumDurationMicroseconds, - ars.LastDurationMicroseconds, - ars.TotalCpuMicroseconds, - ars.AverageCpuMicroseconds, - ars.TotalLogicalReads, - ars.AverageLogicalReads, - ars.TotalPhysicalReads, - ars.AveragePhysicalReads, - ars.TotalLogicalWrites, - ars.AverageLogicalWrites, - ars.AverageRowCount, - ars.MaximumRowCount, - ars.FirstExecutionTimeUtc, - ars.LastExecutionTimeUtc, - q.count_compiles AS QueryLevelCompileCount, - SWITCHOFFSET(q.last_compile_start_time, '+00:00') AS QueryLevelLastCompileTimeUtc, - p.is_forced_plan AS IsForcedPlan, - p.force_failure_count AS ForceFailureCount, - p.last_force_failure_reason AS LastForceFailureReason, - p.last_force_failure_reason_desc AS LastForceFailureReasonDescription, - p.plan_group_id AS PlanGroupId, - p.engine_version AS EngineVersion, - p.compatibility_level AS CompatibilityLevel, - p.is_online_index_plan AS IsOnlineIndexPlan, - p.is_trivial_plan AS IsTrivialPlan, - p.is_parallel_plan AS IsParallelPlan, - CASE - WHEN @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) - END AS TotalWaitMilliseconds, - CASE - WHEN @WaitStatsStatus = 'Available' THEN CONVERT(decimal(38, 4), ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) - END AS AverageWaitMilliseconds, - @WaitStatsStatus AS WaitStatsStatus, - CASE - WHEN @WaitStatsStatus = 'Available' THEN ISNULL(wsx.WaitStatsXml, CONVERT(xml, N'')) - END AS WaitStatsXml - FROM AggregatedRuntimeStats AS ars - INNER JOIN sys.query_store_plan AS p - ON p.plan_id = ars.PlanId - INNER JOIN sys.query_store_query AS q - ON q.query_id = p.query_id - INNER JOIN sys.query_store_query_text AS qt - ON qt.query_text_id = q.query_text_id - LEFT JOIN AggregatedWaitStats AS aws - ON aws.PlanId = ars.PlanId - LEFT JOIN WaitStatsXml AS wsx - ON wsx.PlanId = ars.PlanId - -- Self-exclusion keeps these diagnostic procedures' own Query Store entries out of their own - -- results, so running diagnostics does not appear as a "slow query" in the output. - WHERE ISNULL(q.object_id, -1) <> ISNULL(@SlowQueriesProcedureObjectId, -2) - AND ISNULL(q.object_id, -1) <> ISNULL(@PlanDiagnosticsProcedureObjectId, -2) - AND ISNULL(q.object_id, -1) <> ISNULL(@StatisticsHealthProcedureObjectId, -2) - AND (@QueryTextContains IS NULL OR qt.query_sql_text LIKE @QueryTextPattern ESCAPE N'~') - -- The requested @OrderBy column drives the primary sort key and every other CASE branch - -- evaluates to NULL, so ties still fall back to query_id/plan_id for a stable, deterministic order. - ORDER BY - CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' AND aws.TotalWaitMilliseconds IS NULL THEN 1 ELSE 0 END ASC, - CASE WHEN @OrderByNormalized = 'TotalDuration' THEN ars.TotalDurationMicroseconds END DESC, - CASE WHEN @OrderByNormalized = 'AverageDuration' THEN ars.AverageDurationMicroseconds END DESC, - CASE WHEN @OrderByNormalized = 'MaximumDuration' THEN ars.MaximumDurationMicroseconds END DESC, - CASE WHEN @OrderByNormalized = 'TotalCpu' THEN ars.TotalCpuMicroseconds END DESC, - CASE WHEN @OrderByNormalized = 'AverageCpu' THEN ars.AverageCpuMicroseconds END DESC, - CASE WHEN @OrderByNormalized = 'LogicalReads' THEN ars.TotalLogicalReads END DESC, - CASE WHEN @OrderByNormalized = 'Executions' THEN ars.RegularExecutionCount END DESC, - CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) END DESC, - q.query_id ASC, - p.plan_id ASC - OFFSET @Offset ROWS FETCH NEXT @Top ROWS ONLY; - - SET @RowsReturned = @@ROWCOUNT; - - EXECUTE dbo.LogEvent - @Process = @ProcedureName, - @Mode = @AuditMode, - @Status = 'End', - @Rows = @RowsReturned, - @Start = @AuditStartTime, - @Text = @AuditText; - END TRY - BEGIN CATCH - -- ------------------------------------------------------------------------ - -- Audit the failure and rethrow - -- ------------------------------------------------------------------------ - SET @AuditText = CONCAT( - @AuditText, - N';ErrorNumber=', ERROR_NUMBER(), - N';ErrorState=', ERROR_STATE()); - - EXECUTE dbo.LogEvent - @Process = @ProcedureName, - @Mode = @AuditMode, - @Status = 'Error', - @Start = @AuditStartTime, - @Text = @AuditText; - - THROW; - END CATCH -END -GO - -CREATE OR ALTER PROCEDURE dbo.GetQueryStorePlanDiagnostics @PlanId bigint -WITH EXECUTE AS 'dbo' -AS -BEGIN - SET NOCOUNT ON - - DECLARE @SP varchar(100) = OBJECT_NAME(@@PROCID) - ,@Mode varchar(200) = 'QueryStorePlanDiagnostics' - ,@Start datetime = GETUTCDATE() - ,@Rows int = 0 - ,@QueryStoreState nvarchar(60) - ,@AuditText nvarchar(3500) - ,@FoundPlanId bigint - ,@QueryId bigint - ,@QueryHash binary(8) - ,@QueryPlanHash binary(8) - ,@QuerySqlText nvarchar(max) - ,@ObjectId int - ,@PlanGroupId bigint - ,@EngineVersion nvarchar(128) - ,@CompatibilityLevel smallint - ,@IsOnlineIndexPlan bit - ,@IsTrivialPlan bit - ,@IsParallelPlan bit - ,@IsForcedPlan bit - ,@ForceFailureCount bigint - ,@LastForceFailureReason int - ,@LastForceFailureReasonDesc nvarchar(256) - ,@CountCompiles bigint - ,@InitialCompileStartTime datetimeoffset(7) - ,@LastCompileStartTime datetimeoffset(7) - ,@LastPlanExecutionTime datetimeoffset(7) - ,@AverageCompileDuration float - ,@LastCompileDuration bigint - ,@FirstExecutionTime datetimeoffset(7) - ,@LastRuntimeExecutionTime datetimeoffset(7) - ,@RawQueryPlan nvarchar(max) - ,@LocalPlanXml xml - ,@SanitizedShowPlanXml xml - ,@SerializedPlanXml nvarchar(max) - ,@RemainingParameterListCount bigint - ,@ForbiddenAttributeCount bigint - ,@SanitizationStatus varchar(32) - ,@SanitizationErrorCode varchar(64) - ,@SerializedResultSizeBytes bigint = 0 - ,@CaughtErrorNumber int - ,@CaughtErrorState int - - SET @AuditText = CONCAT( - N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), - N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), - N';PlanId=', ISNULL(CONVERT(nvarchar(20), @PlanId), N'NULL')) - - BEGIN TRY - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Start',@Text=@AuditText - - -- ------------------------------------------------------------------------ - -- Validate parameters and prerequisite Query Store state - -- ------------------------------------------------------------------------ - IF @PlanId IS NULL OR @PlanId <= 0 - THROW 50001, 'Plan ID must be a positive bigint.', 1 - - SELECT @QueryStoreState = actual_state_desc - FROM sys.database_query_store_options - - IF @QueryStoreState IS NULL OR @QueryStoreState NOT IN ('READ_WRITE', 'READ_ONLY') - THROW 50002, 'Query Store is not readable.', 1 - - -- ------------------------------------------------------------------------ - -- Look up the plan and collect metadata safely into local variables - -- ------------------------------------------------------------------------ - -- Projecting into scalar variables (rather than selecting directly) keeps the raw, unsanitized - -- query_plan XML out of the result set contract while it is still being sanitized below. - SELECT @FoundPlanId = p.plan_id - ,@QueryId = p.query_id - ,@QueryHash = q.query_hash - ,@QueryPlanHash = p.query_plan_hash - ,@QuerySqlText = qt.query_sql_text - ,@ObjectId = q.object_id - ,@PlanGroupId = p.plan_group_id - ,@EngineVersion = p.engine_version - ,@CompatibilityLevel = p.compatibility_level - ,@IsOnlineIndexPlan = p.is_online_index_plan - ,@IsTrivialPlan = p.is_trivial_plan - ,@IsParallelPlan = p.is_parallel_plan - ,@IsForcedPlan = p.is_forced_plan - ,@ForceFailureCount = p.force_failure_count - ,@LastForceFailureReason = p.last_force_failure_reason - ,@LastForceFailureReasonDesc = p.last_force_failure_reason_desc - ,@CountCompiles = p.count_compiles - ,@InitialCompileStartTime = p.initial_compile_start_time - ,@LastCompileStartTime = p.last_compile_start_time - ,@LastPlanExecutionTime = p.last_execution_time - ,@AverageCompileDuration = p.avg_compile_duration - ,@LastCompileDuration = p.last_compile_duration - ,@RawQueryPlan = CONVERT(nvarchar(max), p.query_plan) - FROM sys.query_store_plan AS p - LEFT JOIN sys.query_store_query AS q - ON q.query_id = p.query_id - LEFT JOIN sys.query_store_query_text AS qt - ON qt.query_text_id = q.query_text_id - WHERE p.plan_id = @PlanId - - IF @FoundPlanId IS NULL - THROW 50003, 'The requested Query Store plan was not found or is no longer retained.', 1 - - SELECT @FirstExecutionTime = MIN(first_execution_time) - ,@LastRuntimeExecutionTime = MAX(last_execution_time) - FROM sys.query_store_runtime_stats - WHERE plan_id = @PlanId - - -- ------------------------------------------------------------------------ - -- Sanitize the Showplan XML and verify the ParameterList removal - -- ------------------------------------------------------------------------ - -- Raw or partially sanitized plans are never returned to the caller: this branch either produces - -- a verified-clean plan or leaves @SanitizedShowPlanXml NULL with a status/error code explaining why. - IF @RawQueryPlan IS NULL - BEGIN - SET @SanitizationStatus = 'PlanXmlUnavailable' - SET @SanitizationErrorCode = 'PLAN_XML_UNAVAILABLE' - END - ELSE - BEGIN - SET @LocalPlanXml = TRY_CONVERT(xml, @RawQueryPlan) - - IF @LocalPlanXml IS NULL - BEGIN - SET @SanitizationStatus = 'InvalidXml' - SET @SanitizationErrorCode = 'PLAN_XML_INVALID' - END - ELSE - BEGIN - BEGIN TRY - SET @SanitizedShowPlanXml = @LocalPlanXml - -- local-name() matches the ParameterList element regardless of the Showplan XML - -- namespace/version, since Query Store XML namespaces can vary across engine versions. - SET @SanitizedShowPlanXml.modify('delete //*[local-name(.) = "ParameterList"]') - - -- Defense-in-depth verification: check both the structural XML (no remaining - -- ParameterList elements/attributes) and the serialized text (no leftover literal - -- tokens) before trusting the sanitized plan is safe to return. - SET @RemainingParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') - SET @ForbiddenAttributeCount = @SanitizedShowPlanXml.value('count(//@*[local-name(.) = "ParameterCompiledValue" or local-name(.) = "ParameterRuntimeValue"])', 'bigint') - SET @SerializedPlanXml = CONVERT(nvarchar(max), @SanitizedShowPlanXml) - - IF @RemainingParameterListCount = 0 - AND @ForbiddenAttributeCount = 0 - AND CHARINDEX(N'PARAMETERLIST', UPPER(@SerializedPlanXml)) = 0 - AND CHARINDEX(N'PARAMETERCOMPILEDVALUE', UPPER(@SerializedPlanXml)) = 0 - AND CHARINDEX(N'PARAMETERRUNTIMEVALUE', UPPER(@SerializedPlanXml)) = 0 - BEGIN - SET @SanitizationStatus = 'Sanitized' - END - ELSE - BEGIN - SET @SanitizedShowPlanXml = NULL - SET @SerializedPlanXml = NULL - SET @SanitizationStatus = 'VerificationFailed' - SET @SanitizationErrorCode = 'PLAN_XML_VERIFICATION_FAILED' - END - END TRY - BEGIN CATCH - SET @SanitizedShowPlanXml = NULL - SET @SerializedPlanXml = NULL - SET @SanitizationStatus = 'VerificationFailed' - SET @SanitizationErrorCode = 'PLAN_XML_VERIFICATION_FAILED' - END CATCH - END - END - - SET @SerializedResultSizeBytes = ISNULL(DATALENGTH(@SerializedPlanXml), 0) - SET @AuditText = CONCAT( - @AuditText, - N';SanitizationStatus=', ISNULL(CONVERT(nvarchar(32), @SanitizationStatus), N'NotStarted'), - N';SanitizationErrorCode=', ISNULL(CONVERT(nvarchar(64), @SanitizationErrorCode), N'NONE'), - N';SerializedResultSizeBytes=', CONVERT(nvarchar(20), @SerializedResultSizeBytes)) - - IF @SanitizationErrorCode IS NOT NULL - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Text=@AuditText - - SET @Rows = 1 - - -- ------------------------------------------------------------------------ - -- Project the plan metadata and verified Showplan XML - -- ------------------------------------------------------------------------ - SELECT @FoundPlanId AS PlanId - ,@QueryId AS QueryId - ,@QueryHash AS QueryHash - ,@QueryPlanHash AS QueryPlanHash - ,@QuerySqlText AS QuerySqlText - ,@ObjectId AS ObjectId - ,@PlanGroupId AS PlanGroupId - ,@EngineVersion AS EngineVersion - ,@CompatibilityLevel AS CompatibilityLevel - ,@CountCompiles AS CompileCount - ,@InitialCompileStartTime AS InitialCompileStartTime - ,@LastCompileStartTime AS LastCompileStartTime - ,@AverageCompileDuration AS AverageCompileDurationMicroseconds - ,@LastCompileDuration AS LastCompileDurationMicroseconds - ,@IsOnlineIndexPlan AS IsOnlineIndexPlan - ,@IsTrivialPlan AS IsTrivialPlan - ,@IsParallelPlan AS IsParallelPlan - ,@IsForcedPlan AS IsForcedPlan - ,@ForceFailureCount AS ForceFailureCount - ,@LastForceFailureReason AS LastForceFailureReason - ,@LastForceFailureReasonDesc AS LastForceFailureReasonDescription - ,@FirstExecutionTime AS FirstExecutionTime - ,COALESCE(@LastRuntimeExecutionTime, @LastPlanExecutionTime) AS LastExecutionTime - ,@SanitizationStatus AS SanitizationStatus - ,@SanitizationErrorCode AS SanitizationErrorCode - ,@SanitizedShowPlanXml AS SanitizedShowPlanXml - - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@Start,@Rows=@Rows,@Text=@AuditText - END TRY - BEGIN CATCH - SET @CaughtErrorNumber = ERROR_NUMBER() - SET @CaughtErrorState = ERROR_STATE() - -- ------------------------------------------------------------------------ - -- Audit the failure and rethrow - -- ------------------------------------------------------------------------ - SET @AuditText = CONCAT( - N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), - N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), - N';PlanId=', ISNULL(CONVERT(nvarchar(20), @PlanId), N'NULL'), - N';SanitizationStatus=', ISNULL(CONVERT(nvarchar(32), @SanitizationStatus), N'NotCompleted'), - N';SanitizationErrorCode=PLAN_DIAGNOSTICS_FAILED', - N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), - N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)) - - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@Start,@Text=@AuditText; - THROW; - END CATCH -END -GO - -CREATE OR ALTER PROCEDURE dbo.GetStatisticsHealth - @TableName nvarchar(128) = NULL, - @Top int = 20, - @Offset int = 0, - @OrderBy varchar(32) = 'ModificationPercent' -WITH EXECUTE AS 'dbo' -AS -BEGIN - SET NOCOUNT ON; - - DECLARE @SP varchar(100) = OBJECT_NAME(@@PROCID); - DECLARE @Mode varchar(200) = 'StatisticsHealth'; - DECLARE @AuditText nvarchar(3500) = CONCAT( - N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), - N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), - N';TableNameSupplied=', CASE WHEN @TableName IS NULL THEN N'0' ELSE N'1' END, - N';Top=', ISNULL(CONVERT(nvarchar(11), @Top), N'NULL'), - N';Offset=', ISNULL(CONVERT(nvarchar(11), @Offset), N'NULL')); - DECLARE @Start datetime = GETUTCDATE(); - DECLARE @Rows int; - DECLARE @TableObjectId int; - DECLARE @TableCount int; - DECLARE @NormalizedOrderBy varchar(32) = UPPER(@OrderBy COLLATE Latin1_General_100_CI_AS); - DECLARE @CaughtErrorNumber int; - DECLARE @CaughtErrorState int; - - BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @AuditText; - - -- ------------------------------------------------------------------------ - -- Validate parameters - -- ------------------------------------------------------------------------ - IF @Top IS NULL OR @Top < 1 OR @Top > 100 - BEGIN - THROW 50000, '@Top must be between 1 and 100.', 127; - END - - IF @Offset IS NULL OR @Offset < 0 OR @Offset > 10000 - BEGIN - THROW 50000, '@Offset must be between 0 and 10000.', 127; - END - - IF @NormalizedOrderBy IS NULL - OR @NormalizedOrderBy NOT IN ('MODIFICATIONCOUNT', 'MODIFICATIONPERCENT', 'LASTUPDATED', 'SAMPLINGPERCENT', 'ROWS') - BEGIN - THROW 50000, '@OrderBy must be ModificationCount, ModificationPercent, LastUpdated, SamplingPercent, or Rows.', 127; - END - - IF @TableName IS NOT NULL - BEGIN - IF LEN(LTRIM(RTRIM(@TableName))) = 0 - BEGIN - THROW 50000, '@TableName must be nonblank when supplied.', 127; - END - - -- ------------------------------------------------------------------------ - -- Resolve the requested table name to exactly one user table - -- ------------------------------------------------------------------------ - SELECT - @TableCount = COUNT(*), - @TableObjectId = MIN(tableInfo.object_id) - FROM sys.tables AS tableInfo - WHERE tableInfo.is_ms_shipped = 0 - AND tableInfo.name COLLATE DATABASE_DEFAULT = @TableName COLLATE DATABASE_DEFAULT; - - IF @TableCount = 0 - BEGIN - THROW 50000, '@TableName does not resolve to a user table.', 127; - END - - IF @TableCount > 1 - BEGIN - THROW 50000, '@TableName must resolve to exactly one user table.', 127; - END - END - - SET @AuditText = CONCAT( - @AuditText, - N';TableName=', ISNULL(@TableName, N'NULL'), - N';OrderBy=', @NormalizedOrderBy); - - -- ------------------------------------------------------------------------ - -- Project statistics-column metadata as XML alongside stats/index properties - -- ------------------------------------------------------------------------ - ;WITH StatisticsMetadata AS - ( - SELECT - tableInfo.name AS TableName, - statisticsInfo.name AS StatisticsName, - statisticsInfo.stats_id AS StatisticsId, - ( - SELECT - statisticsColumn.stats_column_id AS [@Ordinal], - columnInfo.name AS [@Name] - FROM sys.stats_columns AS statisticsColumn - INNER JOIN sys.columns AS columnInfo - ON columnInfo.object_id = statisticsColumn.object_id - AND columnInfo.column_id = statisticsColumn.column_id - WHERE statisticsColumn.object_id = statisticsInfo.object_id - AND statisticsColumn.stats_id = statisticsInfo.stats_id - ORDER BY statisticsColumn.stats_column_id - FOR XML PATH('StatisticsColumn'), ROOT('StatisticsColumns'), TYPE - ) AS StatisticsColumns, - statisticsInfo.auto_created AS AutoCreated, - statisticsInfo.user_created AS UserCreated, - statisticsInfo.is_incremental AS IsIncremental, - statisticsInfo.has_persisted_sample AS HasPersistedSample, - statisticsInfo.no_recompute AS NoRecompute, - statisticsInfo.has_filter AS HasFilter, - statisticsInfo.filter_definition AS FilterDefinition, - indexInfo.index_id AS IndexId, - indexInfo.name AS IndexName, - indexInfo.type_desc AS IndexTypeDescription, - indexInfo.is_disabled AS IsIndexDisabled, - indexInfo.is_hypothetical AS IsIndexHypothetical, - statisticsProperties.last_updated AS LastUpdated, - CONVERT(decimal(38, 4), - CONVERT(decimal(38, 0), DATEDIFF_BIG(SECOND, statisticsProperties.last_updated, SYSUTCDATETIME())) - / CONVERT(decimal(4, 0), 3600)) AS HoursSinceLastUpdate, - statisticsProperties.rows AS [Rows], - statisticsProperties.unfiltered_rows AS UnfilteredRows, - statisticsProperties.rows_sampled AS RowsSampled, - -- NULLIF guards a zero-row denominator (returns NULL instead of a divide-by-zero error), and - -- a percentage can legitimately exceed 100 (e.g. rows_sampled/modification_counter can outgrow - -- a stale rows count), so the result is not clamped. - CONVERT(decimal(38, 4), - (CONVERT(decimal(38, 0), statisticsProperties.rows_sampled) * CONVERT(decimal(3, 0), 100)) - / NULLIF(CONVERT(decimal(38, 0), statisticsProperties.rows), CONVERT(decimal(38, 0), 0))) AS SamplingPercent, - statisticsProperties.steps AS HistogramStepCount, - statisticsProperties.modification_counter AS ModificationCount, - CONVERT(decimal(38, 4), - (CONVERT(decimal(38, 0), statisticsProperties.modification_counter) * CONVERT(decimal(3, 0), 100)) - / NULLIF(CONVERT(decimal(38, 0), statisticsProperties.rows), CONVERT(decimal(38, 0), 0))) AS ModificationPercent, - CASE - WHEN statisticsProperties.PropertiesAvailable IS NULL THEN 'PropertiesUnavailable' - ELSE 'Available' - END AS StatisticsStatus - FROM sys.tables AS tableInfo - INNER JOIN sys.stats AS statisticsInfo - ON statisticsInfo.object_id = tableInfo.object_id - LEFT JOIN sys.indexes AS indexInfo - ON indexInfo.object_id = statisticsInfo.object_id - AND indexInfo.index_id = statisticsInfo.stats_id - -- OUTER APPLY (rather than CROSS APPLY) preserves the statistics row even when - -- sys.dm_db_stats_properties returns nothing, e.g. for an unsupported/inaccessible object; - -- StatisticsStatus below reports 'PropertiesUnavailable' instead of silently dropping the row. - OUTER APPLY - ( - SELECT - 1 AS PropertiesAvailable, - properties.last_updated, - properties.rows, - properties.rows_sampled, - properties.steps, - properties.unfiltered_rows, - properties.modification_counter - FROM sys.dm_db_stats_properties(statisticsInfo.object_id, statisticsInfo.stats_id) AS properties - ) AS statisticsProperties - WHERE tableInfo.is_ms_shipped = 0 - AND - ( - (@TableName IS NOT NULL AND tableInfo.object_id = @TableObjectId) - -- Without an explicit @TableName, temporal history tables (temporal_type = 1) are - -- excluded because their statistics mirror the corresponding current table. - OR - (@TableName IS NULL AND tableInfo.temporal_type <> 1) - ) - ), - OrderedStatisticsMetadata AS - ( - SELECT - *, - ROW_NUMBER() OVER - ( - ORDER BY - CASE WHEN @NormalizedOrderBy = 'MODIFICATIONCOUNT' THEN ModificationCount END DESC, - CASE WHEN @NormalizedOrderBy = 'MODIFICATIONPERCENT' THEN ModificationPercent END DESC, - -- NULL LastUpdated (properties unavailable) sorts first regardless of ASC/DESC, - -- then the actual timestamp orders the known values oldest-first. - CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' AND LastUpdated IS NULL THEN 0 - WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN 1 - END ASC, - CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN LastUpdated END ASC, - CASE WHEN @NormalizedOrderBy = 'SAMPLINGPERCENT' THEN SamplingPercent END DESC, - CASE WHEN @NormalizedOrderBy = 'ROWS' THEN [Rows] END DESC, - -- Table/statistics identity is the final, always-present tie-breaker so paging - -- is deterministic regardless of which @OrderBy column is requested. - TableName ASC, - StatisticsName ASC, - StatisticsId ASC - ) AS RowNumber - FROM StatisticsMetadata - ) - -- ------------------------------------------------------------------------ - -- Project results and apply offset/top paging over the deterministic order - -- ------------------------------------------------------------------------ - SELECT - TableName, - StatisticsName, - StatisticsId, - StatisticsColumns, - AutoCreated, - UserCreated, - IsIncremental, - HasPersistedSample, - NoRecompute, - HasFilter, - FilterDefinition, - IndexId, - IndexName, - IndexTypeDescription, - IsIndexDisabled, - IsIndexHypothetical, - LastUpdated, - HoursSinceLastUpdate, - [Rows], - UnfilteredRows, - RowsSampled, - SamplingPercent, - HistogramStepCount, - ModificationCount, - ModificationPercent, - StatisticsStatus - FROM OrderedStatisticsMetadata - WHERE RowNumber > @Offset - AND RowNumber <= @Offset + @Top - ORDER BY RowNumber; - - SET @Rows = @@ROWCOUNT; - - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @Start, @Rows = @Rows, @Text = @AuditText; - END TRY - BEGIN CATCH - SET @CaughtErrorNumber = ERROR_NUMBER(); - SET @CaughtErrorState = ERROR_STATE(); - -- ------------------------------------------------------------------------ - -- Audit the failure and rethrow - -- ------------------------------------------------------------------------ - SET @AuditText = CONCAT( - N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), - N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), - N';StatisticsHealthErrorCode=STATISTICS_HEALTH_FAILED', - N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), - N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)); - - -- Real error is before 1750, cannot trap in SQL; rethrow immediately without attempting to audit. - IF ERROR_NUMBER() = 1750 THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @Start, @Text = @AuditText; - THROW; - END CATCH -END -GO - --- ── 3. Grant EXECUTE on each procedure individually to [FhirDiagnosticsReader] - -GRANT EXECUTE ON dbo.GetQueryStoreSlowQueries TO [FhirDiagnosticsReader]; -GRANT EXECUTE ON dbo.GetQueryStorePlanDiagnostics TO [FhirDiagnosticsReader]; -GRANT EXECUTE ON dbo.GetStatisticsHealth TO [FhirDiagnosticsReader]; -GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs index 0642d918dc..340aebb356 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs @@ -126,6 +126,5 @@ public enum SchemaVersion V114 = 114, V115 = 115, V116 = 116, - V117 = 117, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index ee91e20469..854d345f6b 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema public static class SchemaVersionConstants { public const int Min = (int)SchemaVersion.V113; - public const int Max = (int)SchemaVersion.V117; + public const int Max = (int)SchemaVersion.V116; public const int MinForUpgrade = (int)SchemaVersion.V111; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/DiagnosticsRole.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/DiagnosticsRole.sql deleted file mode 100644 index 11d05a39c8..0000000000 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/DiagnosticsRole.sql +++ /dev/null @@ -1,26 +0,0 @@ -IF NOT EXISTS -( - SELECT 1 - FROM sys.database_principals - WHERE name = N'FhirDiagnosticsReader' - AND type = 'R' -) -BEGIN - IF EXISTS - ( - SELECT 1 - FROM sys.database_principals - WHERE name = N'FhirDiagnosticsReader' - ) - BEGIN - THROW 50100, 'A database principal named FhirDiagnosticsReader already exists but is not a database role.', 1; - END - - CREATE ROLE [FhirDiagnosticsReader]; -END -GO - -GRANT EXECUTE ON dbo.GetQueryStoreSlowQueries TO [FhirDiagnosticsReader]; -GRANT EXECUTE ON dbo.GetQueryStorePlanDiagnostics TO [FhirDiagnosticsReader]; -GRANT EXECUTE ON dbo.GetStatisticsHealth TO [FhirDiagnosticsReader]; -GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql deleted file mode 100644 index d248bfbc2d..0000000000 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStorePlanDiagnostics.sql +++ /dev/null @@ -1,236 +0,0 @@ ---DROP PROCEDURE dbo.GetQueryStorePlanDiagnostics -GO -CREATE PROCEDURE dbo.GetQueryStorePlanDiagnostics @PlanId bigint -WITH EXECUTE AS 'dbo' -AS -BEGIN - SET NOCOUNT ON - - DECLARE @SP varchar(100) = OBJECT_NAME(@@PROCID) - ,@Mode varchar(200) = 'QueryStorePlanDiagnostics' - ,@Start datetime = GETUTCDATE() - ,@Rows int = 0 - ,@QueryStoreState nvarchar(60) - ,@AuditText nvarchar(3500) - ,@FoundPlanId bigint - ,@QueryId bigint - ,@QueryHash binary(8) - ,@QueryPlanHash binary(8) - ,@QuerySqlText nvarchar(max) - ,@ObjectId int - ,@PlanGroupId bigint - ,@EngineVersion nvarchar(128) - ,@CompatibilityLevel smallint - ,@IsOnlineIndexPlan bit - ,@IsTrivialPlan bit - ,@IsParallelPlan bit - ,@IsForcedPlan bit - ,@ForceFailureCount bigint - ,@LastForceFailureReason int - ,@LastForceFailureReasonDesc nvarchar(256) - ,@CountCompiles bigint - ,@InitialCompileStartTime datetimeoffset(7) - ,@LastCompileStartTime datetimeoffset(7) - ,@LastPlanExecutionTime datetimeoffset(7) - ,@AverageCompileDuration float - ,@LastCompileDuration bigint - ,@FirstExecutionTime datetimeoffset(7) - ,@LastRuntimeExecutionTime datetimeoffset(7) - ,@RawQueryPlan nvarchar(max) - ,@LocalPlanXml xml - ,@SanitizedShowPlanXml xml - ,@SerializedPlanXml nvarchar(max) - ,@RemainingParameterListCount bigint - ,@ForbiddenAttributeCount bigint - ,@SanitizationStatus varchar(32) - ,@SanitizationErrorCode varchar(64) - ,@SerializedResultSizeBytes bigint = 0 - ,@CaughtErrorNumber int - ,@CaughtErrorState int - - SET @AuditText = CONCAT( - N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), - N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), - N';PlanId=', ISNULL(CONVERT(nvarchar(20), @PlanId), N'NULL')) - - BEGIN TRY - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Start',@Text=@AuditText - - -- ------------------------------------------------------------------------ - -- Validate parameters and prerequisite Query Store state - -- ------------------------------------------------------------------------ - IF @PlanId IS NULL OR @PlanId <= 0 - THROW 50001, 'Plan ID must be a positive bigint.', 1 - - SELECT @QueryStoreState = actual_state_desc - FROM sys.database_query_store_options - - IF @QueryStoreState IS NULL OR @QueryStoreState NOT IN ('READ_WRITE', 'READ_ONLY') - THROW 50002, 'Query Store is not readable.', 1 - - -- ------------------------------------------------------------------------ - -- Look up the plan and collect metadata safely into local variables - -- ------------------------------------------------------------------------ - -- Projecting into scalar variables (rather than selecting directly) keeps the raw, unsanitized - -- query_plan XML out of the result set contract while it is still being sanitized below. - SELECT @FoundPlanId = p.plan_id - ,@QueryId = p.query_id - ,@QueryHash = q.query_hash - ,@QueryPlanHash = p.query_plan_hash - ,@QuerySqlText = qt.query_sql_text - ,@ObjectId = q.object_id - ,@PlanGroupId = p.plan_group_id - ,@EngineVersion = p.engine_version - ,@CompatibilityLevel = p.compatibility_level - ,@IsOnlineIndexPlan = p.is_online_index_plan - ,@IsTrivialPlan = p.is_trivial_plan - ,@IsParallelPlan = p.is_parallel_plan - ,@IsForcedPlan = p.is_forced_plan - ,@ForceFailureCount = p.force_failure_count - ,@LastForceFailureReason = p.last_force_failure_reason - ,@LastForceFailureReasonDesc = p.last_force_failure_reason_desc - ,@CountCompiles = p.count_compiles - ,@InitialCompileStartTime = p.initial_compile_start_time - ,@LastCompileStartTime = p.last_compile_start_time - ,@LastPlanExecutionTime = p.last_execution_time - ,@AverageCompileDuration = p.avg_compile_duration - ,@LastCompileDuration = p.last_compile_duration - ,@RawQueryPlan = CONVERT(nvarchar(max), p.query_plan) - FROM sys.query_store_plan AS p - LEFT JOIN sys.query_store_query AS q - ON q.query_id = p.query_id - LEFT JOIN sys.query_store_query_text AS qt - ON qt.query_text_id = q.query_text_id - WHERE p.plan_id = @PlanId - - IF @FoundPlanId IS NULL - THROW 50003, 'The requested Query Store plan was not found or is no longer retained.', 1 - - SELECT @FirstExecutionTime = MIN(first_execution_time) - ,@LastRuntimeExecutionTime = MAX(last_execution_time) - FROM sys.query_store_runtime_stats - WHERE plan_id = @PlanId - - -- ------------------------------------------------------------------------ - -- Sanitize the Showplan XML and verify the ParameterList removal - -- ------------------------------------------------------------------------ - -- Raw or partially sanitized plans are never returned to the caller: this branch either produces - -- a verified-clean plan or leaves @SanitizedShowPlanXml NULL with a status/error code explaining why. - IF @RawQueryPlan IS NULL - BEGIN - SET @SanitizationStatus = 'PlanXmlUnavailable' - SET @SanitizationErrorCode = 'PLAN_XML_UNAVAILABLE' - END - ELSE - BEGIN - SET @LocalPlanXml = TRY_CONVERT(xml, @RawQueryPlan) - - IF @LocalPlanXml IS NULL - BEGIN - SET @SanitizationStatus = 'InvalidXml' - SET @SanitizationErrorCode = 'PLAN_XML_INVALID' - END - ELSE - BEGIN - BEGIN TRY - SET @SanitizedShowPlanXml = @LocalPlanXml - -- local-name() matches the ParameterList element regardless of the Showplan XML - -- namespace/version, since Query Store XML namespaces can vary across engine versions. - SET @SanitizedShowPlanXml.modify('delete //*[local-name(.) = "ParameterList"]') - - -- Defense-in-depth verification: check both the structural XML (no remaining - -- ParameterList elements/attributes) and the serialized text (no leftover literal - -- tokens) before trusting the sanitized plan is safe to return. - SET @RemainingParameterListCount = @SanitizedShowPlanXml.value('count(//*[local-name(.) = "ParameterList"])', 'bigint') - SET @ForbiddenAttributeCount = @SanitizedShowPlanXml.value('count(//@*[local-name(.) = "ParameterCompiledValue" or local-name(.) = "ParameterRuntimeValue"])', 'bigint') - SET @SerializedPlanXml = CONVERT(nvarchar(max), @SanitizedShowPlanXml) - - IF @RemainingParameterListCount = 0 - AND @ForbiddenAttributeCount = 0 - AND CHARINDEX(N'PARAMETERLIST', UPPER(@SerializedPlanXml)) = 0 - AND CHARINDEX(N'PARAMETERCOMPILEDVALUE', UPPER(@SerializedPlanXml)) = 0 - AND CHARINDEX(N'PARAMETERRUNTIMEVALUE', UPPER(@SerializedPlanXml)) = 0 - BEGIN - SET @SanitizationStatus = 'Sanitized' - END - ELSE - BEGIN - SET @SanitizedShowPlanXml = NULL - SET @SerializedPlanXml = NULL - SET @SanitizationStatus = 'VerificationFailed' - SET @SanitizationErrorCode = 'PLAN_XML_VERIFICATION_FAILED' - END - END TRY - BEGIN CATCH - SET @SanitizedShowPlanXml = NULL - SET @SerializedPlanXml = NULL - SET @SanitizationStatus = 'VerificationFailed' - SET @SanitizationErrorCode = 'PLAN_XML_VERIFICATION_FAILED' - END CATCH - END - END - - SET @SerializedResultSizeBytes = ISNULL(DATALENGTH(@SerializedPlanXml), 0) - SET @AuditText = CONCAT( - @AuditText, - N';SanitizationStatus=', ISNULL(CONVERT(nvarchar(32), @SanitizationStatus), N'NotStarted'), - N';SanitizationErrorCode=', ISNULL(CONVERT(nvarchar(64), @SanitizationErrorCode), N'NONE'), - N';SerializedResultSizeBytes=', CONVERT(nvarchar(20), @SerializedResultSizeBytes)) - - IF @SanitizationErrorCode IS NOT NULL - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Text=@AuditText - - SET @Rows = 1 - - -- ------------------------------------------------------------------------ - -- Project the plan metadata and verified Showplan XML - -- ------------------------------------------------------------------------ - SELECT @FoundPlanId AS PlanId - ,@QueryId AS QueryId - ,@QueryHash AS QueryHash - ,@QueryPlanHash AS QueryPlanHash - ,@QuerySqlText AS QuerySqlText - ,@ObjectId AS ObjectId - ,@PlanGroupId AS PlanGroupId - ,@EngineVersion AS EngineVersion - ,@CompatibilityLevel AS CompatibilityLevel - ,@CountCompiles AS CompileCount - ,@InitialCompileStartTime AS InitialCompileStartTime - ,@LastCompileStartTime AS LastCompileStartTime - ,@AverageCompileDuration AS AverageCompileDurationMicroseconds - ,@LastCompileDuration AS LastCompileDurationMicroseconds - ,@IsOnlineIndexPlan AS IsOnlineIndexPlan - ,@IsTrivialPlan AS IsTrivialPlan - ,@IsParallelPlan AS IsParallelPlan - ,@IsForcedPlan AS IsForcedPlan - ,@ForceFailureCount AS ForceFailureCount - ,@LastForceFailureReason AS LastForceFailureReason - ,@LastForceFailureReasonDesc AS LastForceFailureReasonDescription - ,@FirstExecutionTime AS FirstExecutionTime - ,COALESCE(@LastRuntimeExecutionTime, @LastPlanExecutionTime) AS LastExecutionTime - ,@SanitizationStatus AS SanitizationStatus - ,@SanitizationErrorCode AS SanitizationErrorCode - ,@SanitizedShowPlanXml AS SanitizedShowPlanXml - - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@Start,@Rows=@Rows,@Text=@AuditText - END TRY - BEGIN CATCH - SET @CaughtErrorNumber = ERROR_NUMBER() - SET @CaughtErrorState = ERROR_STATE() - -- ------------------------------------------------------------------------ - -- Audit the failure and rethrow - -- ------------------------------------------------------------------------ - SET @AuditText = CONCAT( - N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), - N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), - N';PlanId=', ISNULL(CONVERT(nvarchar(20), @PlanId), N'NULL'), - N';SanitizationStatus=', ISNULL(CONVERT(nvarchar(32), @SanitizationStatus), N'NotCompleted'), - N';SanitizationErrorCode=PLAN_DIAGNOSTICS_FAILED', - N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), - N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)) - - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@Start,@Text=@AuditText; - THROW; - END CATCH -END -GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql deleted file mode 100644 index 7a3919ffaa..0000000000 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetQueryStoreSlowQueries.sql +++ /dev/null @@ -1,430 +0,0 @@ ---DROP PROCEDURE dbo.GetQueryStoreSlowQueries -GO -CREATE PROCEDURE dbo.GetQueryStoreSlowQueries - @StartTime datetimeoffset(7) = NULL - ,@EndTime datetimeoffset(7) = NULL - ,@Top int = 20 - ,@Offset int = 0 - ,@OrderBy varchar(32) = 'TotalDuration' - ,@MinExecutions bigint = 1 - ,@QueryTextContains nvarchar(256) = NULL -WITH EXECUTE AS 'dbo' -AS -BEGIN - SET NOCOUNT ON; - - DECLARE @ProcedureName varchar(100) = OBJECT_NAME(@@PROCID); - DECLARE @AuditMode varchar(200) = 'QueryStoreSlowQueries'; - DECLARE @AuditStartTime datetime = GETUTCDATE(); - DECLARE @AuditText nvarchar(3500); - DECLARE @RowsReturned bigint; - DECLARE @ResolvedStartTime datetimeoffset(7); - DECLARE @ResolvedEndTime datetimeoffset(7); - DECLARE @OrderByNormalized varchar(32); - DECLARE @QueryTextPattern nvarchar(514); - DECLARE @QueryTextFilterLength int; - DECLARE @QueryStoreState nvarchar(60); - DECLARE @QueryStoreReadOnlyReason bigint; - DECLARE @WaitStatsCaptureMode nvarchar(60); - DECLARE @WaitStatsStatus varchar(32); - DECLARE @SlowQueriesProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStoreSlowQueries'); - DECLARE @PlanDiagnosticsProcedureObjectId int = OBJECT_ID(N'dbo.GetQueryStorePlanDiagnostics'); - DECLARE @StatisticsHealthProcedureObjectId int = OBJECT_ID(N'dbo.GetStatisticsHealth'); - - IF @ProcedureName IS NULL - SET @ProcedureName = 'GetQueryStoreSlowQueries'; - - SET @AuditText = CONCAT( - N'OriginalLogin=', ORIGINAL_LOGIN(), - N';EffectivePrincipal=', USER_NAME()); - - BEGIN TRY - -- ------------------------------------------------------------------------ - -- Resolve defaults, capture Query Store state, and build the audit context - -- ------------------------------------------------------------------------ - SET @Top = ISNULL(@Top, 20); - SET @Offset = ISNULL(@Offset, 0); - SET @MinExecutions = ISNULL(@MinExecutions, 1); - SET @OrderBy = ISNULL(@OrderBy, 'TotalDuration'); - SET @ResolvedEndTime = SWITCHOFFSET(ISNULL(@EndTime, TODATETIMEOFFSET(SYSUTCDATETIME(), '+00:00')), '+00:00'); - SET @ResolvedStartTime = SWITCHOFFSET(ISNULL(@StartTime, DATEADD(hour, -1, @ResolvedEndTime)), '+00:00'); - SET @QueryTextContains = NULLIF(LTRIM(RTRIM(@QueryTextContains)), N''); - SET @QueryTextFilterLength = ISNULL(LEN(@QueryTextContains), 0); - - SELECT - @QueryStoreState = actual_state_desc, - @QueryStoreReadOnlyReason = readonly_reason, - @WaitStatsCaptureMode = wait_stats_capture_mode_desc - FROM sys.database_query_store_options; - - SET @WaitStatsStatus = - CASE @WaitStatsCaptureMode - WHEN N'ON' THEN 'Available' - WHEN N'OFF' THEN 'Disabled' - ELSE 'Unavailable' - END; - SET @AuditText = CONCAT( - @AuditText, - N';StartTimeUtc=', CONVERT(nvarchar(33), @ResolvedStartTime, 127), - N';EndTimeUtc=', CONVERT(nvarchar(33), @ResolvedEndTime, 127), - N';OrderBy=', @OrderBy, - N';Top=', @Top, - N';Offset=', @Offset, - N';MinExecutions=', @MinExecutions, - N';QueryTextFilterPresent=', CASE WHEN @QueryTextContains IS NULL THEN N'0' ELSE N'1' END, - N';QueryTextFilterLength=', @QueryTextFilterLength, - N';QueryStoreState=', ISNULL(@QueryStoreState, N'Unknown'), - N';QueryStoreReadOnlyReason=', ISNULL(CONVERT(nvarchar(20), @QueryStoreReadOnlyReason), N'Unknown'), - N';WaitStatsCaptureMode=', ISNULL(@WaitStatsCaptureMode, N'Unknown')); - - EXECUTE dbo.LogEvent - @Process = @ProcedureName, - @Mode = @AuditMode, - @Status = 'Start', - @Text = @AuditText; - - -- ------------------------------------------------------------------------ - -- Validate parameters - -- ------------------------------------------------------------------------ - IF @Top < 1 OR @Top > 100 - THROW 50400, '@Top must be between 1 and 100.', 1; - - IF @Offset < 0 OR @Offset > 10000 - THROW 50401, '@Offset must be between 0 and 10000.', 1; - - IF @MinExecutions < 1 - THROW 50402, '@MinExecutions must be positive.', 1; - - IF @ResolvedStartTime >= @ResolvedEndTime - THROW 50403, '@StartTime must precede @EndTime.', 1; - - IF @ResolvedEndTime > DATEADD(hour, 24, @ResolvedStartTime) - THROW 50404, 'The requested time range must not exceed 24 hours.', 1; - - IF @QueryTextContains IS NOT NULL - AND LEN(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(@QueryTextContains, N' ', N''), NCHAR(9), N''), NCHAR(10), N''), NCHAR(13), N''), NCHAR(160), N'')) = 0 - THROW 50405, '@QueryTextContains must not be whitespace only.', 1; - - IF @QueryTextContains IS NOT NULL - AND (@QueryTextFilterLength < 3 OR @QueryTextFilterLength > 256) - THROW 50406, '@QueryTextContains must contain between 3 and 256 characters after trimming.', 1; - - SET @OrderByNormalized = - CASE - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalDuration' THEN 'TotalDuration' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageDuration' THEN 'AverageDuration' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'MaximumDuration' THEN 'MaximumDuration' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalCpu' THEN 'TotalCpu' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'AverageCpu' THEN 'AverageCpu' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'LogicalReads' THEN 'LogicalReads' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'Executions' THEN 'Executions' - WHEN @OrderBy COLLATE Latin1_General_100_CI_AS = 'TotalWait' THEN 'TotalWait' - END; - - IF @OrderByNormalized IS NULL - THROW 50407, '@OrderBy is not supported.', 1; - - -- ------------------------------------------------------------------------ - -- Validate Query Store prerequisite state - -- ------------------------------------------------------------------------ - IF ISNULL(@QueryStoreState, N'') NOT IN (N'READ_WRITE', N'READ_ONLY') - THROW 50408, 'Query Store is not enabled and readable.', 1; - - -- ------------------------------------------------------------------------ - -- Normalize the literal filter into a safe LIKE pattern - -- ------------------------------------------------------------------------ - -- Escape wildcard/escape characters before wrapping so a literal '%', '_', or '[' in the - -- caller-supplied text is matched literally rather than interpreted by LIKE. - SET @QueryTextPattern = - CASE - WHEN @QueryTextContains IS NULL THEN NULL - ELSE N'%' + REPLACE(REPLACE(REPLACE(REPLACE(@QueryTextContains, N'~', N'~~'), N'%', N'~%'), N'_', N'~_'), N'[', N'~[') + N'%' - END; - - -- ------------------------------------------------------------------------ - -- Collapse duplicate runtime-stats rows and compute weighted aggregates - -- ------------------------------------------------------------------------ - -- Active Query Store intervals can expose both a persisted row and an in-memory row for the - -- same plan/interval, so duplicates must be collapsed before aggregating across intervals. - ;WITH RuntimeStatsRows AS - ( - SELECT - rs.plan_id AS PlanId, - rs.execution_type AS ExecutionType, - rs.runtime_stats_interval_id AS RuntimeStatsIntervalId, - rs.runtime_stats_id AS RuntimeStatsId, - rs.count_executions AS RegularExecutionCount, - CONVERT(decimal(38, 4), rs.avg_duration) AS AverageDurationMicroseconds, - CONVERT(decimal(38, 0), rs.min_duration) AS MinimumDurationMicroseconds, - CONVERT(decimal(38, 0), rs.max_duration) AS MaximumDurationMicroseconds, - CONVERT(decimal(38, 0), rs.last_duration) AS LastDurationMicroseconds, - CONVERT(decimal(38, 4), rs.avg_cpu_time) AS AverageCpuMicroseconds, - CONVERT(decimal(38, 4), rs.avg_logical_io_reads) AS AverageLogicalReads, - CONVERT(decimal(38, 4), rs.avg_physical_io_reads) AS AveragePhysicalReads, - CONVERT(decimal(38, 4), rs.avg_logical_io_writes) AS AverageLogicalWrites, - CONVERT(decimal(38, 4), rs.avg_rowcount) AS AverageRowCount, - CONVERT(decimal(38, 0), rs.max_rowcount) AS MaximumRowCount, - SWITCHOFFSET(rs.first_execution_time, '+00:00') AS FirstExecutionTimeUtc, - SWITCHOFFSET(rs.last_execution_time, '+00:00') AS LastExecutionTimeUtc - FROM sys.query_store_runtime_stats AS rs - INNER JOIN sys.query_store_runtime_stats_interval AS rsi - ON rsi.runtime_stats_interval_id = rs.runtime_stats_interval_id - -- The baseline is regular executions only. An explicit execution-type input belongs here if added later. - WHERE rs.execution_type = 0 - -- Query Store interval overlap semantics are inclusive of edge executions, so a run that only - -- partially overlaps the requested [@ResolvedStartTime, @ResolvedEndTime) window is still included. - AND rsi.start_time < @ResolvedEndTime - AND rsi.end_time > @ResolvedStartTime - ), - RankedRuntimeStatsRows AS - ( - SELECT - rs.*, - ROW_NUMBER() OVER - ( - PARTITION BY rs.PlanId, rs.ExecutionType, rs.RuntimeStatsIntervalId - ORDER BY rs.LastExecutionTimeUtc DESC, rs.RuntimeStatsIntervalId DESC, rs.RuntimeStatsId DESC - ) AS LastValueRank - FROM RuntimeStatsRows AS rs - ), - CollapsedRuntimeStats AS - ( - SELECT - rs.PlanId, - rs.ExecutionType, - rs.RuntimeStatsIntervalId, - SUM(rs.RegularExecutionCount) AS RegularExecutionCount, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageDurationMicroseconds * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalDurationMicroseconds, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageCpuMicroseconds * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalCpuMicroseconds, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageLogicalReads * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalLogicalReads, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AveragePhysicalReads * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalPhysicalReads, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageLogicalWrites * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalLogicalWrites, - CONVERT(decimal(38, 4), SUM(CONVERT(decimal(38, 4), rs.AverageRowCount * CONVERT(decimal(19, 0), rs.RegularExecutionCount)))) AS TotalRowCount, - MIN(rs.MinimumDurationMicroseconds) AS MinimumDurationMicroseconds, - MAX(rs.MaximumDurationMicroseconds) AS MaximumDurationMicroseconds, - MAX(rs.MaximumRowCount) AS MaximumRowCount, - MIN(rs.FirstExecutionTimeUtc) AS FirstExecutionTimeUtc, - MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastExecutionTimeUtc END) AS LastExecutionTimeUtc, - MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastDurationMicroseconds END) AS LastDurationMicroseconds, - MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.RuntimeStatsId END) AS LastRuntimeStatsId - FROM RankedRuntimeStatsRows AS rs - GROUP BY - rs.PlanId, - rs.ExecutionType, - rs.RuntimeStatsIntervalId - ), - RankedCollapsedRuntimeStats AS - ( - SELECT - rs.*, - ROW_NUMBER() OVER - ( - PARTITION BY rs.PlanId - ORDER BY rs.LastExecutionTimeUtc DESC, rs.RuntimeStatsIntervalId DESC, rs.LastRuntimeStatsId DESC - ) AS LastValueRank - FROM CollapsedRuntimeStats AS rs - ), - AggregatedRuntimeStats AS - ( - SELECT - rs.PlanId, - SUM(rs.RegularExecutionCount) AS RegularExecutionCount, - CONVERT(decimal(38, 0), SUM(rs.TotalDurationMicroseconds)) AS TotalDurationMicroseconds, - -- Averages are recomputed as execution-count-weighted sums rather than averaged directly, - -- and division uses decimal precision so intervals with unequal execution counts are not skewed. - CONVERT(decimal(38, 4), SUM(rs.TotalDurationMicroseconds) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageDurationMicroseconds, - MIN(rs.MinimumDurationMicroseconds) AS MinimumDurationMicroseconds, - MAX(rs.MaximumDurationMicroseconds) AS MaximumDurationMicroseconds, - MAX(CASE WHEN rs.LastValueRank = 1 THEN rs.LastDurationMicroseconds END) AS LastDurationMicroseconds, - CONVERT(decimal(38, 0), SUM(rs.TotalCpuMicroseconds)) AS TotalCpuMicroseconds, - CONVERT(decimal(38, 4), SUM(rs.TotalCpuMicroseconds) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageCpuMicroseconds, - CONVERT(decimal(38, 0), SUM(rs.TotalLogicalReads)) AS TotalLogicalReads, - CONVERT(decimal(38, 4), SUM(rs.TotalLogicalReads) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageLogicalReads, - CONVERT(decimal(38, 0), SUM(rs.TotalPhysicalReads)) AS TotalPhysicalReads, - CONVERT(decimal(38, 4), SUM(rs.TotalPhysicalReads) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AveragePhysicalReads, - CONVERT(decimal(38, 0), SUM(rs.TotalLogicalWrites)) AS TotalLogicalWrites, - CONVERT(decimal(38, 4), SUM(rs.TotalLogicalWrites) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageLogicalWrites, - CONVERT(decimal(38, 4), SUM(rs.TotalRowCount) / NULLIF(CONVERT(decimal(38, 4), SUM(rs.RegularExecutionCount)), CONVERT(decimal(38, 4), 0))) AS AverageRowCount, - MAX(rs.MaximumRowCount) AS MaximumRowCount, - MIN(rs.FirstExecutionTimeUtc) AS FirstExecutionTimeUtc, - MAX(rs.LastExecutionTimeUtc) AS LastExecutionTimeUtc - FROM RankedCollapsedRuntimeStats AS rs - GROUP BY rs.PlanId - HAVING SUM(rs.RegularExecutionCount) >= @MinExecutions - ), - -- ------------------------------------------------------------------------ - -- Aggregate wait-stat rows and handle capture availability - -- ------------------------------------------------------------------------ - WaitStatsRows AS - ( - SELECT - ws.plan_id AS PlanId, - ws.wait_category_desc AS WaitCategoryDescription, - CONVERT(decimal(38, 0), ws.total_query_wait_time_ms) AS TotalWaitMilliseconds, - CONVERT(decimal(38, 0), ws.max_query_wait_time_ms) AS MaximumWaitMilliseconds - FROM sys.query_store_wait_stats AS ws - INNER JOIN sys.query_store_runtime_stats_interval AS rsi - ON rsi.runtime_stats_interval_id = ws.runtime_stats_interval_id - -- When wait capture is disabled or unavailable this CTE is intentionally left empty so runtime - -- rows are still returned with NULL wait columns instead of failing the whole query. - WHERE @WaitStatsStatus = 'Available' - AND ws.execution_type = 0 - AND rsi.start_time < @ResolvedEndTime - AND rsi.end_time > @ResolvedStartTime - ), - WaitCategories AS - ( - SELECT - ws.PlanId, - ws.WaitCategoryDescription, - CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds, - MAX(ws.MaximumWaitMilliseconds) AS MaximumWaitMilliseconds - FROM WaitStatsRows AS ws - GROUP BY - ws.PlanId, - ws.WaitCategoryDescription - ), - AggregatedWaitStats AS - ( - SELECT - ws.PlanId, - CONVERT(decimal(38, 0), SUM(ws.TotalWaitMilliseconds)) AS TotalWaitMilliseconds - FROM WaitCategories AS ws - GROUP BY ws.PlanId - ), - WaitStatsXml AS - ( - SELECT - wp.PlanId, - ( - SELECT - wc.WaitCategoryDescription AS [@Category], - wc.TotalWaitMilliseconds AS [@TotalWaitMilliseconds], - CONVERT(decimal(38, 4), wc.TotalWaitMilliseconds / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) AS [@AverageWaitMilliseconds], - wc.MaximumWaitMilliseconds AS [@MaximumWaitMilliseconds] - FROM WaitCategories AS wc - WHERE wc.PlanId = wp.PlanId - AND wc.TotalWaitMilliseconds > 0 - ORDER BY - wc.TotalWaitMilliseconds DESC, - wc.WaitCategoryDescription ASC - FOR XML PATH(N'WaitCategory'), ROOT(N'WaitStats'), TYPE - ) AS WaitStatsXml - FROM (SELECT DISTINCT PlanId FROM WaitCategories) AS wp - INNER JOIN AggregatedRuntimeStats AS ars - ON ars.PlanId = wp.PlanId - ) - -- ------------------------------------------------------------------------ - -- Project results with static, deterministic ordering - -- ------------------------------------------------------------------------ - SELECT - q.query_id AS QueryId, - p.plan_id AS PlanId, - q.query_hash AS QueryHash, - p.query_plan_hash AS QueryPlanHash, - qt.query_sql_text AS QuerySqlText, - q.object_id AS ObjectId, - OBJECT_NAME(q.object_id) AS ObjectName, - ars.RegularExecutionCount, - ars.TotalDurationMicroseconds, - ars.AverageDurationMicroseconds, - ars.MinimumDurationMicroseconds, - ars.MaximumDurationMicroseconds, - ars.LastDurationMicroseconds, - ars.TotalCpuMicroseconds, - ars.AverageCpuMicroseconds, - ars.TotalLogicalReads, - ars.AverageLogicalReads, - ars.TotalPhysicalReads, - ars.AveragePhysicalReads, - ars.TotalLogicalWrites, - ars.AverageLogicalWrites, - ars.AverageRowCount, - ars.MaximumRowCount, - ars.FirstExecutionTimeUtc, - ars.LastExecutionTimeUtc, - q.count_compiles AS QueryLevelCompileCount, - SWITCHOFFSET(q.last_compile_start_time, '+00:00') AS QueryLevelLastCompileTimeUtc, - p.is_forced_plan AS IsForcedPlan, - p.force_failure_count AS ForceFailureCount, - p.last_force_failure_reason AS LastForceFailureReason, - p.last_force_failure_reason_desc AS LastForceFailureReasonDescription, - p.plan_group_id AS PlanGroupId, - p.engine_version AS EngineVersion, - p.compatibility_level AS CompatibilityLevel, - p.is_online_index_plan AS IsOnlineIndexPlan, - p.is_trivial_plan AS IsTrivialPlan, - p.is_parallel_plan AS IsParallelPlan, - CASE - WHEN @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) - END AS TotalWaitMilliseconds, - CASE - WHEN @WaitStatsStatus = 'Available' THEN CONVERT(decimal(38, 4), ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) / NULLIF(CONVERT(decimal(38, 4), ars.RegularExecutionCount), CONVERT(decimal(38, 4), 0))) - END AS AverageWaitMilliseconds, - @WaitStatsStatus AS WaitStatsStatus, - CASE - WHEN @WaitStatsStatus = 'Available' THEN ISNULL(wsx.WaitStatsXml, CONVERT(xml, N'')) - END AS WaitStatsXml - FROM AggregatedRuntimeStats AS ars - INNER JOIN sys.query_store_plan AS p - ON p.plan_id = ars.PlanId - INNER JOIN sys.query_store_query AS q - ON q.query_id = p.query_id - INNER JOIN sys.query_store_query_text AS qt - ON qt.query_text_id = q.query_text_id - LEFT JOIN AggregatedWaitStats AS aws - ON aws.PlanId = ars.PlanId - LEFT JOIN WaitStatsXml AS wsx - ON wsx.PlanId = ars.PlanId - -- Self-exclusion keeps these diagnostic procedures' own Query Store entries out of their own - -- results, so running diagnostics does not appear as a "slow query" in the output. - WHERE ISNULL(q.object_id, -1) <> ISNULL(@SlowQueriesProcedureObjectId, -2) - AND ISNULL(q.object_id, -1) <> ISNULL(@PlanDiagnosticsProcedureObjectId, -2) - AND ISNULL(q.object_id, -1) <> ISNULL(@StatisticsHealthProcedureObjectId, -2) - AND (@QueryTextContains IS NULL OR qt.query_sql_text LIKE @QueryTextPattern ESCAPE N'~') - -- The requested @OrderBy column drives the primary sort key and every other CASE branch - -- evaluates to NULL, so ties still fall back to query_id/plan_id for a stable, deterministic order. - ORDER BY - CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' AND aws.TotalWaitMilliseconds IS NULL THEN 1 ELSE 0 END ASC, - CASE WHEN @OrderByNormalized = 'TotalDuration' THEN ars.TotalDurationMicroseconds END DESC, - CASE WHEN @OrderByNormalized = 'AverageDuration' THEN ars.AverageDurationMicroseconds END DESC, - CASE WHEN @OrderByNormalized = 'MaximumDuration' THEN ars.MaximumDurationMicroseconds END DESC, - CASE WHEN @OrderByNormalized = 'TotalCpu' THEN ars.TotalCpuMicroseconds END DESC, - CASE WHEN @OrderByNormalized = 'AverageCpu' THEN ars.AverageCpuMicroseconds END DESC, - CASE WHEN @OrderByNormalized = 'LogicalReads' THEN ars.TotalLogicalReads END DESC, - CASE WHEN @OrderByNormalized = 'Executions' THEN ars.RegularExecutionCount END DESC, - CASE WHEN @OrderByNormalized = 'TotalWait' AND @WaitStatsStatus = 'Available' THEN ISNULL(aws.TotalWaitMilliseconds, CONVERT(decimal(38, 0), 0)) END DESC, - q.query_id ASC, - p.plan_id ASC - OFFSET @Offset ROWS FETCH NEXT @Top ROWS ONLY; - - SET @RowsReturned = @@ROWCOUNT; - - EXECUTE dbo.LogEvent - @Process = @ProcedureName, - @Mode = @AuditMode, - @Status = 'End', - @Rows = @RowsReturned, - @Start = @AuditStartTime, - @Text = @AuditText; - END TRY - BEGIN CATCH - -- ------------------------------------------------------------------------ - -- Audit the failure and rethrow - -- ------------------------------------------------------------------------ - SET @AuditText = CONCAT( - @AuditText, - N';ErrorNumber=', ERROR_NUMBER(), - N';ErrorState=', ERROR_STATE()); - - EXECUTE dbo.LogEvent - @Process = @ProcedureName, - @Mode = @AuditMode, - @Status = 'Error', - @Start = @AuditStartTime, - @Text = @AuditText; - - THROW; - END CATCH -END -GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql deleted file mode 100644 index 407d12aa2f..0000000000 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetStatisticsHealth.sql +++ /dev/null @@ -1,252 +0,0 @@ -CREATE OR ALTER PROCEDURE dbo.GetStatisticsHealth - @TableName nvarchar(128) = NULL, - @Top int = 20, - @Offset int = 0, - @OrderBy varchar(32) = 'ModificationPercent' -WITH EXECUTE AS 'dbo' -AS -BEGIN - SET NOCOUNT ON; - - DECLARE @SP varchar(100) = OBJECT_NAME(@@PROCID); - DECLARE @Mode varchar(200) = 'StatisticsHealth'; - DECLARE @AuditText nvarchar(3500) = CONCAT( - N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), - N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), - N';TableNameSupplied=', CASE WHEN @TableName IS NULL THEN N'0' ELSE N'1' END, - N';Top=', ISNULL(CONVERT(nvarchar(11), @Top), N'NULL'), - N';Offset=', ISNULL(CONVERT(nvarchar(11), @Offset), N'NULL')); - DECLARE @Start datetime = GETUTCDATE(); - DECLARE @Rows int; - DECLARE @TableObjectId int; - DECLARE @TableCount int; - DECLARE @NormalizedOrderBy varchar(32) = UPPER(@OrderBy COLLATE Latin1_General_100_CI_AS); - DECLARE @CaughtErrorNumber int; - DECLARE @CaughtErrorState int; - - BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @AuditText; - - -- ------------------------------------------------------------------------ - -- Validate parameters - -- ------------------------------------------------------------------------ - IF @Top IS NULL OR @Top < 1 OR @Top > 100 - BEGIN - THROW 50000, '@Top must be between 1 and 100.', 127; - END - - IF @Offset IS NULL OR @Offset < 0 OR @Offset > 10000 - BEGIN - THROW 50000, '@Offset must be between 0 and 10000.', 127; - END - - IF @NormalizedOrderBy IS NULL - OR @NormalizedOrderBy NOT IN ('MODIFICATIONCOUNT', 'MODIFICATIONPERCENT', 'LASTUPDATED', 'SAMPLINGPERCENT', 'ROWS') - BEGIN - THROW 50000, '@OrderBy must be ModificationCount, ModificationPercent, LastUpdated, SamplingPercent, or Rows.', 127; - END - - IF @TableName IS NOT NULL - BEGIN - IF LEN(LTRIM(RTRIM(@TableName))) = 0 - BEGIN - THROW 50000, '@TableName must be nonblank when supplied.', 127; - END - - -- ------------------------------------------------------------------------ - -- Resolve the requested table name to exactly one user table - -- ------------------------------------------------------------------------ - SELECT - @TableCount = COUNT(*), - @TableObjectId = MIN(tableInfo.object_id) - FROM sys.tables AS tableInfo - WHERE tableInfo.is_ms_shipped = 0 - AND tableInfo.name COLLATE DATABASE_DEFAULT = @TableName COLLATE DATABASE_DEFAULT; - - IF @TableCount = 0 - BEGIN - THROW 50000, '@TableName does not resolve to a user table.', 127; - END - - IF @TableCount > 1 - BEGIN - THROW 50000, '@TableName must resolve to exactly one user table.', 127; - END - END - - SET @AuditText = CONCAT( - @AuditText, - N';TableName=', ISNULL(@TableName, N'NULL'), - N';OrderBy=', @NormalizedOrderBy); - - -- ------------------------------------------------------------------------ - -- Project statistics-column metadata as XML alongside stats/index properties - -- ------------------------------------------------------------------------ - ;WITH StatisticsMetadata AS - ( - SELECT - tableInfo.name AS TableName, - statisticsInfo.name AS StatisticsName, - statisticsInfo.stats_id AS StatisticsId, - ( - SELECT - statisticsColumn.stats_column_id AS [@Ordinal], - columnInfo.name AS [@Name] - FROM sys.stats_columns AS statisticsColumn - INNER JOIN sys.columns AS columnInfo - ON columnInfo.object_id = statisticsColumn.object_id - AND columnInfo.column_id = statisticsColumn.column_id - WHERE statisticsColumn.object_id = statisticsInfo.object_id - AND statisticsColumn.stats_id = statisticsInfo.stats_id - ORDER BY statisticsColumn.stats_column_id - FOR XML PATH('StatisticsColumn'), ROOT('StatisticsColumns'), TYPE - ) AS StatisticsColumns, - statisticsInfo.auto_created AS AutoCreated, - statisticsInfo.user_created AS UserCreated, - statisticsInfo.is_incremental AS IsIncremental, - statisticsInfo.has_persisted_sample AS HasPersistedSample, - statisticsInfo.no_recompute AS NoRecompute, - statisticsInfo.has_filter AS HasFilter, - statisticsInfo.filter_definition AS FilterDefinition, - indexInfo.index_id AS IndexId, - indexInfo.name AS IndexName, - indexInfo.type_desc AS IndexTypeDescription, - indexInfo.is_disabled AS IsIndexDisabled, - indexInfo.is_hypothetical AS IsIndexHypothetical, - statisticsProperties.last_updated AS LastUpdated, - CONVERT(decimal(38, 4), - CONVERT(decimal(38, 0), DATEDIFF_BIG(SECOND, statisticsProperties.last_updated, SYSUTCDATETIME())) - / CONVERT(decimal(4, 0), 3600)) AS HoursSinceLastUpdate, - statisticsProperties.rows AS [Rows], - statisticsProperties.unfiltered_rows AS UnfilteredRows, - statisticsProperties.rows_sampled AS RowsSampled, - -- NULLIF guards a zero-row denominator (returns NULL instead of a divide-by-zero error), and - -- a percentage can legitimately exceed 100 (e.g. rows_sampled/modification_counter can outgrow - -- a stale rows count), so the result is not clamped. - CONVERT(decimal(38, 4), - (CONVERT(decimal(38, 0), statisticsProperties.rows_sampled) * CONVERT(decimal(3, 0), 100)) - / NULLIF(CONVERT(decimal(38, 0), statisticsProperties.rows), CONVERT(decimal(38, 0), 0))) AS SamplingPercent, - statisticsProperties.steps AS HistogramStepCount, - statisticsProperties.modification_counter AS ModificationCount, - CONVERT(decimal(38, 4), - (CONVERT(decimal(38, 0), statisticsProperties.modification_counter) * CONVERT(decimal(3, 0), 100)) - / NULLIF(CONVERT(decimal(38, 0), statisticsProperties.rows), CONVERT(decimal(38, 0), 0))) AS ModificationPercent, - CASE - WHEN statisticsProperties.PropertiesAvailable IS NULL THEN 'PropertiesUnavailable' - ELSE 'Available' - END AS StatisticsStatus - FROM sys.tables AS tableInfo - INNER JOIN sys.stats AS statisticsInfo - ON statisticsInfo.object_id = tableInfo.object_id - LEFT JOIN sys.indexes AS indexInfo - ON indexInfo.object_id = statisticsInfo.object_id - AND indexInfo.index_id = statisticsInfo.stats_id - -- OUTER APPLY (rather than CROSS APPLY) preserves the statistics row even when - -- sys.dm_db_stats_properties returns nothing, e.g. for an unsupported/inaccessible object; - -- StatisticsStatus below reports 'PropertiesUnavailable' instead of silently dropping the row. - OUTER APPLY - ( - SELECT - 1 AS PropertiesAvailable, - properties.last_updated, - properties.rows, - properties.rows_sampled, - properties.steps, - properties.unfiltered_rows, - properties.modification_counter - FROM sys.dm_db_stats_properties(statisticsInfo.object_id, statisticsInfo.stats_id) AS properties - ) AS statisticsProperties - WHERE tableInfo.is_ms_shipped = 0 - AND - ( - (@TableName IS NOT NULL AND tableInfo.object_id = @TableObjectId) - -- Without an explicit @TableName, temporal history tables (temporal_type = 1) are - -- excluded because their statistics mirror the corresponding current table. - OR - (@TableName IS NULL AND tableInfo.temporal_type <> 1) - ) - ), - OrderedStatisticsMetadata AS - ( - SELECT - *, - ROW_NUMBER() OVER - ( - ORDER BY - CASE WHEN @NormalizedOrderBy = 'MODIFICATIONCOUNT' THEN ModificationCount END DESC, - CASE WHEN @NormalizedOrderBy = 'MODIFICATIONPERCENT' THEN ModificationPercent END DESC, - -- NULL LastUpdated (properties unavailable) sorts first regardless of ASC/DESC, - -- then the actual timestamp orders the known values oldest-first. - CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' AND LastUpdated IS NULL THEN 0 - WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN 1 - END ASC, - CASE WHEN @NormalizedOrderBy = 'LASTUPDATED' THEN LastUpdated END ASC, - CASE WHEN @NormalizedOrderBy = 'SAMPLINGPERCENT' THEN SamplingPercent END DESC, - CASE WHEN @NormalizedOrderBy = 'ROWS' THEN [Rows] END DESC, - -- Table/statistics identity is the final, always-present tie-breaker so paging - -- is deterministic regardless of which @OrderBy column is requested. - TableName ASC, - StatisticsName ASC, - StatisticsId ASC - ) AS RowNumber - FROM StatisticsMetadata - ) - -- ------------------------------------------------------------------------ - -- Project results and apply offset/top paging over the deterministic order - -- ------------------------------------------------------------------------ - SELECT - TableName, - StatisticsName, - StatisticsId, - StatisticsColumns, - AutoCreated, - UserCreated, - IsIncremental, - HasPersistedSample, - NoRecompute, - HasFilter, - FilterDefinition, - IndexId, - IndexName, - IndexTypeDescription, - IsIndexDisabled, - IsIndexHypothetical, - LastUpdated, - HoursSinceLastUpdate, - [Rows], - UnfilteredRows, - RowsSampled, - SamplingPercent, - HistogramStepCount, - ModificationCount, - ModificationPercent, - StatisticsStatus - FROM OrderedStatisticsMetadata - WHERE RowNumber > @Offset - AND RowNumber <= @Offset + @Top - ORDER BY RowNumber; - - SET @Rows = @@ROWCOUNT; - - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @Start, @Rows = @Rows, @Text = @AuditText; - END TRY - BEGIN CATCH - SET @CaughtErrorNumber = ERROR_NUMBER(); - SET @CaughtErrorState = ERROR_STATE(); - -- ------------------------------------------------------------------------ - -- Audit the failure and rethrow - -- ------------------------------------------------------------------------ - SET @AuditText = CONCAT( - N'OriginalLogin=', CONVERT(nvarchar(128), ORIGINAL_LOGIN()), - N';EffectivePrincipal=', CONVERT(nvarchar(128), USER_NAME()), - N';StatisticsHealthErrorCode=STATISTICS_HEALTH_FAILED', - N';ErrorNumber=', CONVERT(nvarchar(11), @CaughtErrorNumber), - N';ErrorState=', CONVERT(nvarchar(11), @CaughtErrorState)); - - -- Real error is before 1750, cannot trap in SQL; rethrow immediately without attempting to audit. - IF ERROR_NUMBER() = 1750 THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @Start, @Text = @AuditText; - THROW; - END CATCH -END -GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs new file mode 100644 index 0000000000..72b51591f6 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs @@ -0,0 +1,36 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +{ + internal sealed class QueryPlanSanitizationResult + { + internal QueryPlanSanitizationResult(string status, string xml, bool truncated, int originalLength, int sanitizedLength) + { + Status = status; + Xml = xml; + Truncated = truncated; + OriginalLength = originalLength; + SanitizedLength = sanitizedLength; + } + + internal string Status { get; } + + internal string Xml { get; } + + internal bool Truncated { get; } + + /// + /// Gets the length of the raw Showplan XML as read from Query Store. + /// + internal int OriginalLength { get; } + + /// + /// Gets the length of the sanitized Showplan XML before truncation, or zero when sanitization did not produce a document. + /// This is the value to compare against the field cap when is set. + /// + internal int SanitizedLength { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs new file mode 100644 index 0000000000..555d8ea449 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs @@ -0,0 +1,85 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.IO; +using System.Linq; +using System.Xml; +using System.Xml.Linq; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +{ + internal static class QueryPlanSanitizer + { + internal const string SanitizedStatus = "Sanitized"; + internal const string PlanXmlUnavailableStatus = "PlanXmlUnavailable"; + internal const string InvalidXmlStatus = "InvalidXml"; + internal const string VerificationFailedStatus = "VerificationFailed"; + + internal static QueryPlanSanitizationResult Sanitize(string queryPlanXml, int maxLength) + { + if (string.IsNullOrEmpty(queryPlanXml)) + { + return new QueryPlanSanitizationResult(PlanXmlUnavailableStatus, null, false, 0, 0); + } + + try + { + var settings = new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Prohibit, + XmlResolver = null, + }; + + XDocument document; + using (var stringReader = new StringReader(queryPlanXml)) + using (var xmlReader = XmlReader.Create(stringReader, settings)) + { + document = XDocument.Load(xmlReader, LoadOptions.PreserveWhitespace); + } + + var elements = document.Root?.DescendantsAndSelf() + .Where(element => string.Equals(element.Name.LocalName, "ParameterList", StringComparison.OrdinalIgnoreCase)) + .ToList(); + elements?.Remove(); + + var attributes = document.Root?.DescendantsAndSelf() + .Attributes() + .Where(attribute => + string.Equals(attribute.Name.LocalName, "ParameterCompiledValue", StringComparison.OrdinalIgnoreCase) || + string.Equals(attribute.Name.LocalName, "ParameterRuntimeValue", StringComparison.OrdinalIgnoreCase)) + .ToList(); + attributes?.Remove(); + + var sanitizedXml = document.ToString(SaveOptions.DisableFormatting); + var sanitizedLength = sanitizedXml.Length; + if (ContainsSensitiveParameterData(sanitizedXml)) + { + return new QueryPlanSanitizationResult(VerificationFailedStatus, null, false, queryPlanXml.Length, sanitizedLength); + } + + maxLength = Math.Max(0, maxLength); + var truncated = sanitizedXml.Length > maxLength; + if (truncated) + { + sanitizedXml = sanitizedXml.Substring(0, maxLength); + } + + return new QueryPlanSanitizationResult(SanitizedStatus, sanitizedXml, truncated, queryPlanXml.Length, sanitizedLength); + } + catch (XmlException) + { + return new QueryPlanSanitizationResult(InvalidXmlStatus, null, false, queryPlanXml.Length, 0); + } + } + + private static bool ContainsSensitiveParameterData(string xml) + { + return xml.Contains("ParameterList", StringComparison.OrdinalIgnoreCase) || + xml.Contains("ParameterCompiledValue", StringComparison.OrdinalIgnoreCase) || + xml.Contains("ParameterRuntimeValue", StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs new file mode 100644 index 0000000000..5d647e20d4 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs @@ -0,0 +1,594 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Medino; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Metrics; +using Microsoft.Health.Fhir.SqlServer.Features.Storage; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +{ + internal sealed class QueryStoreDiagnosticsWatchdog : Watchdog + { + internal const int MaxFieldLength = 32 * 1024; + + private const string QueryStoreStateSql = @" +SELECT actual_state_desc, readonly_reason +FROM sys.database_query_store_options;"; + + private const string SlowQueriesSql = @" +-- Query Store duration and CPU values are recorded in microseconds. +-- Aggregate interval averages with their execution counts before converting to milliseconds. +;WITH PlanRuntimeStatistics AS +( + SELECT + runtimeStatistics.plan_id, + SUM(runtimeStatistics.count_executions) AS execution_count, + SUM(CONVERT(float, runtimeStatistics.avg_duration) * runtimeStatistics.count_executions) AS total_duration_microseconds, + MAX(CONVERT(float, runtimeStatistics.max_duration)) AS max_duration_microseconds, + SUM(CONVERT(float, runtimeStatistics.avg_cpu_time) * runtimeStatistics.count_executions) AS total_cpu_microseconds, + SUM(CONVERT(float, runtimeStatistics.avg_logical_io_reads) * runtimeStatistics.count_executions) AS total_logical_reads, + MIN(runtimeStatisticsInterval.start_time) AS interval_start, + MAX(runtimeStatisticsInterval.end_time) AS interval_end + FROM sys.query_store_runtime_stats AS runtimeStatistics + INNER JOIN sys.query_store_runtime_stats_interval AS runtimeStatisticsInterval + ON runtimeStatistics.runtime_stats_interval_id = runtimeStatisticsInterval.runtime_stats_interval_id + WHERE runtimeStatisticsInterval.end_time >= @StartTime + AND runtimeStatistics.execution_type = 0 -- regular completed executions only + GROUP BY runtimeStatistics.plan_id +) +SELECT TOP (@Top) + queryStoreQuery.query_id, + queryStorePlan.plan_id, + runtimeRollup.execution_count, + runtimeRollup.total_duration_microseconds / 1000.0 AS total_duration_milliseconds, + runtimeRollup.total_duration_microseconds / runtimeRollup.execution_count / 1000.0 AS average_duration_milliseconds, + runtimeRollup.max_duration_microseconds / 1000.0 AS max_duration_milliseconds, + runtimeRollup.total_cpu_microseconds / 1000.0 AS total_cpu_milliseconds, + runtimeRollup.total_cpu_microseconds / runtimeRollup.execution_count / 1000.0 AS average_cpu_milliseconds, + runtimeRollup.total_logical_reads, + runtimeRollup.total_logical_reads / runtimeRollup.execution_count AS average_logical_reads, + queryText.query_sql_text, + runtimeRollup.interval_start, + runtimeRollup.interval_end +FROM PlanRuntimeStatistics AS runtimeRollup +INNER JOIN sys.query_store_plan AS queryStorePlan + ON runtimeRollup.plan_id = queryStorePlan.plan_id +INNER JOIN sys.query_store_query AS queryStoreQuery + ON queryStorePlan.query_id = queryStoreQuery.query_id +INNER JOIN sys.query_store_query_text AS queryText + ON queryStoreQuery.query_text_id = queryText.query_text_id +WHERE runtimeRollup.execution_count > 0 + AND runtimeRollup.total_duration_microseconds / runtimeRollup.execution_count / 1000.0 >= @MinDurationMilliseconds + -- SQL Server does not preserve diagnostic marker comments in query_sql_text, so exclude every watchdog statement by catalog name. + -- The explicit case-insensitive collation also keeps the comparison correct on case-sensitive databases. + AND queryText.query_sql_text COLLATE Latin1_General_CI_AS NOT LIKE N'%query_store%' + AND queryText.query_sql_text COLLATE Latin1_General_CI_AS NOT LIKE N'%dm_db_stats_properties%' +ORDER BY + runtimeRollup.total_duration_microseconds DESC, + queryStoreQuery.query_id, + queryStorePlan.plan_id;"; + + private const string WaitStatisticsSql = @" +-- Wait statistics are collected independently so unavailable wait capture does not suppress slow-query metrics. +;WITH WaitsByCategory AS +( + SELECT + waitStatistics.plan_id, + waitStatistics.wait_category_desc, + SUM(CONVERT(float, waitStatistics.total_query_wait_time_ms)) AS total_wait_milliseconds + FROM sys.query_store_wait_stats AS waitStatistics + INNER JOIN sys.query_store_runtime_stats_interval AS runtimeStatisticsInterval + ON waitStatistics.runtime_stats_interval_id = runtimeStatisticsInterval.runtime_stats_interval_id + WHERE runtimeStatisticsInterval.end_time >= @StartTime + AND waitStatistics.execution_type = 0 -- regular completed executions only + AND waitStatistics.plan_id IN + ( + SELECT CONVERT(bigint, [value]) + FROM STRING_SPLIT(@PlanIds, ',') + ) + GROUP BY waitStatistics.plan_id, waitStatistics.wait_category_desc +), +RankedWaits AS +( + SELECT + plan_id, + SUM(total_wait_milliseconds) OVER (PARTITION BY plan_id) AS total_wait_milliseconds, + wait_category_desc, + ROW_NUMBER() OVER + ( + PARTITION BY plan_id + ORDER BY total_wait_milliseconds DESC, wait_category_desc + ) AS wait_category_rank + FROM WaitsByCategory +) +SELECT plan_id, total_wait_milliseconds, wait_category_desc +FROM RankedWaits +WHERE wait_category_rank = 1;"; + + private const string QueryPlansSql = @" +SELECT plan_id, query_plan +FROM sys.query_store_plan +WHERE plan_id IN +( + SELECT CONVERT(bigint, [value]) + FROM STRING_SPLIT(@PlanIds, ',') +);"; + + private const string StatisticsHealthSql = @" +SELECT TOP (@Top) + SCHEMA_NAME(queryObject.schema_id) AS schema_name, + queryObject.name AS table_name, + statisticsObject.name AS statistics_name, + TODATETIMEOFFSET(statisticsProperties.last_updated, '+00:00') AS last_updated, + statisticsProperties.rows, + statisticsProperties.rows_sampled, + statisticsProperties.modification_counter, + CASE + WHEN statisticsProperties.rows IS NULL OR statisticsProperties.rows = 0 THEN NULL + ELSE CONVERT(float, statisticsProperties.modification_counter) * 100.0 / statisticsProperties.rows + END AS modification_percent, + statisticsObject.auto_created, + statisticsObject.user_created, + CONVERT(bit, CASE WHEN queryIndex.index_id IS NULL THEN 0 ELSE 1 END) AS is_from_index, + statisticsObject.has_filter +FROM sys.stats AS statisticsObject +INNER JOIN sys.objects AS queryObject + ON statisticsObject.object_id = queryObject.object_id +INNER JOIN sys.tables AS queryTable + ON queryObject.object_id = queryTable.object_id +LEFT JOIN sys.indexes AS queryIndex + ON statisticsObject.object_id = queryIndex.object_id + AND statisticsObject.stats_id = queryIndex.index_id +-- OUTER APPLY retains statistics when dm_db_stats_properties has no row. +OUTER APPLY sys.dm_db_stats_properties(statisticsObject.object_id, statisticsObject.stats_id) AS statisticsProperties +WHERE queryObject.is_ms_shipped = 0 + AND queryObject.type = 'U' + AND queryTable.temporal_type <> 1 -- exclude temporal history tables +ORDER BY + CASE + WHEN statisticsProperties.rows IS NULL OR statisticsProperties.rows = 0 THEN NULL + ELSE CONVERT(float, statisticsProperties.modification_counter) / statisticsProperties.rows + END DESC, + statisticsProperties.modification_counter DESC, + SCHEMA_NAME(queryObject.schema_id), + queryObject.name, + statisticsObject.name;"; + + private readonly QueryStoreDiagnosticsConfiguration _configuration; + private readonly ILogger _logger; + private readonly IMediator _mediator; + private readonly ISqlRetryService _sqlRetryService; + + public QueryStoreDiagnosticsWatchdog( + ISqlRetryService sqlRetryService, + ILogger logger, + IMediator mediator, + IOptions watchdogConfiguration) + : base(sqlRetryService, logger) + { + _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _mediator = EnsureArg.IsNotNull(mediator, nameof(mediator)); + _configuration = EnsureArg.IsNotNull(watchdogConfiguration?.Value, nameof(watchdogConfiguration)).QueryStoreDiagnostics; + PeriodSec = _configuration.PeriodSec; + } + + internal QueryStoreDiagnosticsWatchdog() + : base() + { + // this is used to get param names for testing + } + + internal string IsEnabledId => $"{Name}.IsEnabled"; + + // Ten minutes allows the lease to recover promptly without expiring during a diagnostics collection. + public override double LeasePeriodSec { get; internal set; } = 600; + + public override bool AllowRebalance { get; internal set; } = true; + + public override double PeriodSec { get; internal set; } = 3600; + + /// + /// Exposes RunWorkAsync for unit testing purposes. + /// + /// The cancellation token. + /// A task representing the asynchronous operation. + internal Task RunWorkForTestingAsync(CancellationToken cancellationToken) => RunWorkAsync(cancellationToken); + + protected override async Task InitAdditionalParamsAsync() + { + await using var command = new SqlCommand(@" +INSERT INTO dbo.Parameters (Id, Number) SELECT @IsEnabledId, 0"); + command.Parameters.AddWithValue("@IsEnabledId", IsEnabledId); + await command.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); + } + + protected override async Task RunWorkAsync(CancellationToken cancellationToken) + { + try + { + if (!_configuration.Enabled) + { + _logger.LogInformation("QueryStoreDiagnosticsWatchdog is disabled by configuration. Exiting..."); + return; + } + + if (!await IsEnabledAsync(cancellationToken)) + { + _logger.LogInformation("QueryStoreDiagnosticsWatchdog is not enabled. Exiting..."); + return; + } + + var lookbackPeriodSec = Math.Clamp(await GetNumberParameterByIdAsync(PeriodSecId, cancellationToken), 60d, 86400d); + var startTime = DateTimeOffset.UtcNow.AddSeconds(-lookbackPeriodSec); + var queryStoreState = await GetQueryStoreStateAsync(cancellationToken); + if (queryStoreState == null || !string.Equals(queryStoreState.ActualState, "READ_WRITE", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: Query Store is unavailable for diagnostics. State={QueryStoreState}, ReadonlyReason={ReadonlyReason}", + queryStoreState?.ActualState ?? "unavailable", + queryStoreState?.ReadonlyReason); + return; + } + + var slowQueries = await GetSlowQueriesAsync(startTime, cancellationToken); + var waitStatistics = await GetWaitStatisticsAsync(startTime, slowQueries, cancellationToken); + + foreach (var slowQuery in slowQueries) + { + waitStatistics.TryGetValue(slowQuery.PlanId, out var wait); + var queryText = Truncate(slowQuery.QueryText); + await _mediator.PublishAsync( + new SlowQueryNotification + { + QueryId = slowQuery.QueryId, + PlanId = slowQuery.PlanId, + ExecutionCount = slowQuery.ExecutionCount, + TotalDurationMilliseconds = slowQuery.TotalDurationMilliseconds, + AverageDurationMilliseconds = slowQuery.AverageDurationMilliseconds, + MaxDurationMilliseconds = slowQuery.MaxDurationMilliseconds, + TotalCpuMilliseconds = slowQuery.TotalCpuMilliseconds, + AverageCpuMilliseconds = slowQuery.AverageCpuMilliseconds, + TotalLogicalReads = slowQuery.TotalLogicalReads, + AverageLogicalReads = slowQuery.AverageLogicalReads, + TotalWaitMilliseconds = wait?.TotalWaitMilliseconds, + AverageWaitMilliseconds = wait == null ? null : wait.TotalWaitMilliseconds / slowQuery.ExecutionCount, + TopWaitCategory = wait?.TopWaitCategory, + QueryText = queryText.Value, + QueryTextTruncated = queryText.Truncated, + QueryTextLength = queryText.OriginalLength, + IntervalStart = slowQuery.IntervalStart, + IntervalEnd = slowQuery.IntervalEnd, + }, + cancellationToken); + } + + if (_configuration.IncludeQueryPlans && slowQueries.Count > 0) + { + await PublishQueryPlansAsync(slowQueries, cancellationToken); + } + + if (_configuration.IncludeStatisticsHealth && _configuration.StatisticsHealthCount > 0) + { + await PublishStatisticsHealthAsync(cancellationToken); + } + } + catch (SqlException ex) when (ex.Number == 208) + { + _logger.LogDebug(ex, "QueryStoreDiagnosticsWatchdog: Query Store diagnostics views are unavailable."); + } + catch (SqlException ex) when (ex.Number == 229 || ex.Number == 262) + { + _logger.LogWarning(ex, "QueryStoreDiagnosticsWatchdog: SQL permissions do not allow diagnostics collection."); + } + } + + private async Task GetQueryStoreStateAsync(CancellationToken cancellationToken) + { + await using var command = new SqlCommand(QueryStoreStateSql); + var states = await _sqlRetryService.ExecuteReaderAsync( + command, + reader => new QueryStoreState( + reader.IsDBNull(0) ? null : reader.GetString(0), + reader.IsDBNull(1) ? (int?)null : reader.GetInt32(1)), + _logger, + "Failed to read Query Store state", + cancellationToken, + isReadOnly: true); + + return states.Count == 0 ? null : states[0]; + } + + private async Task> GetSlowQueriesAsync(DateTimeOffset startTime, CancellationToken cancellationToken) + { + await using var command = new SqlCommand(SlowQueriesSql); + command.Parameters.Add("@StartTime", SqlDbType.DateTimeOffset).Value = startTime; + command.Parameters.Add("@Top", SqlDbType.Int).Value = Math.Max(0, _configuration.SlowQueryCount); + command.Parameters.Add("@MinDurationMilliseconds", SqlDbType.Int).Value = Math.Max(0, _configuration.MinDurationMilliseconds); + + return await _sqlRetryService.ExecuteReaderAsync( + command, + reader => new SlowQueryResult + { + QueryId = reader.GetInt64(0), + PlanId = reader.GetInt64(1), + ExecutionCount = reader.GetInt64(2), + TotalDurationMilliseconds = reader.GetDouble(3), + AverageDurationMilliseconds = reader.GetDouble(4), + MaxDurationMilliseconds = reader.GetDouble(5), + TotalCpuMilliseconds = reader.GetDouble(6), + AverageCpuMilliseconds = reader.GetDouble(7), + TotalLogicalReads = reader.GetDouble(8), + AverageLogicalReads = reader.GetDouble(9), + QueryText = reader.IsDBNull(10) ? string.Empty : reader.GetString(10), + IntervalStart = reader.GetDateTimeOffset(11), + IntervalEnd = reader.GetDateTimeOffset(12), + }, + _logger, + "Failed to read Query Store slow queries", + cancellationToken, + isReadOnly: true); + } + + private async Task> GetWaitStatisticsAsync( + DateTimeOffset startTime, + IReadOnlyList slowQueries, + CancellationToken cancellationToken) + { + if (slowQueries.Count == 0) + { + return new Dictionary(); + } + + try + { + await using var command = new SqlCommand(WaitStatisticsSql); + command.Parameters.Add("@StartTime", SqlDbType.DateTimeOffset).Value = startTime; + command.Parameters.Add("@PlanIds", SqlDbType.NVarChar, -1).Value = string.Join(',', slowQueries.Select(query => query.PlanId)); + + var waits = await _sqlRetryService.ExecuteReaderAsync( + command, + reader => new WaitStatistics( + reader.GetInt64(0), + reader.GetDouble(1), + reader.IsDBNull(2) ? null : reader.GetString(2)), + _logger, + "Failed to read Query Store wait statistics", + cancellationToken, + isReadOnly: true); + + return waits.ToDictionary(wait => wait.PlanId); + } + catch (SqlException ex) + { + _logger.LogDebug(ex, "QueryStoreDiagnosticsWatchdog: Query Store wait statistics are unavailable for this collection."); + return new Dictionary(); + } + } + + private async Task PublishQueryPlansAsync(IReadOnlyList slowQueries, CancellationToken cancellationToken) + { + await using var command = new SqlCommand(QueryPlansSql); + command.Parameters.Add("@PlanIds", SqlDbType.NVarChar, -1).Value = string.Join(',', slowQueries.Select(query => query.PlanId)); + + var plans = await _sqlRetryService.ExecuteReaderAsync( + command, + reader => new QueryPlanResult( + reader.GetInt64(0), + reader.IsDBNull(1) ? null : reader.GetString(1)), + _logger, + "Failed to read Query Store plans", + cancellationToken, + isReadOnly: true); + var plansById = plans.ToDictionary(plan => plan.PlanId); + + foreach (var slowQuery in slowQueries) + { + plansById.TryGetValue(slowQuery.PlanId, out var queryPlan); + var sanitizedPlan = QueryPlanSanitizer.Sanitize(queryPlan?.QueryPlan, MaxFieldLength); + await _mediator.PublishAsync( + new QueryPlanNotification + { + QueryId = slowQuery.QueryId, + PlanId = slowQuery.PlanId, + SanitizedQueryPlan = sanitizedPlan.Xml, + QueryPlanTruncated = sanitizedPlan.Truncated, + OriginalQueryPlanLength = sanitizedPlan.OriginalLength, + SanitizedQueryPlanLength = sanitizedPlan.SanitizedLength, + SanitizationStatus = sanitizedPlan.Status, + }, + cancellationToken); + } + } + + private async Task PublishStatisticsHealthAsync(CancellationToken cancellationToken) + { + await using var command = new SqlCommand(StatisticsHealthSql); + command.Parameters.Add("@Top", SqlDbType.Int).Value = _configuration.StatisticsHealthCount; + + var statisticsHealth = await _sqlRetryService.ExecuteReaderAsync( + command, + reader => new StatisticsHealthResult + { + SchemaName = reader.GetString(0), + TableName = reader.GetString(1), + StatisticsName = reader.GetString(2), + LastUpdated = reader.IsDBNull(3) ? (DateTimeOffset?)null : reader.GetDateTimeOffset(3), + Rows = reader.IsDBNull(4) ? (long?)null : reader.GetInt64(4), + RowsSampled = reader.IsDBNull(5) ? (long?)null : reader.GetInt64(5), + ModificationCounter = reader.IsDBNull(6) ? (long?)null : reader.GetInt64(6), + ModificationPercent = reader.IsDBNull(7) ? (double?)null : reader.GetDouble(7), + IsAutoCreated = reader.GetBoolean(8), + IsUserCreated = reader.GetBoolean(9), + IsFromIndex = reader.GetBoolean(10), + HasFilter = reader.GetBoolean(11), + }, + _logger, + "Failed to read statistics health", + cancellationToken, + isReadOnly: true); + + foreach (var statistic in statisticsHealth) + { + await _mediator.PublishAsync( + new StatisticsHealthNotification + { + SchemaName = statistic.SchemaName, + TableName = statistic.TableName, + StatisticsName = statistic.StatisticsName, + LastUpdated = statistic.LastUpdated, + Rows = statistic.Rows, + RowsSampled = statistic.RowsSampled, + ModificationCounter = statistic.ModificationCounter, + ModificationPercent = statistic.ModificationPercent, + IsAutoCreated = statistic.IsAutoCreated, + IsUserCreated = statistic.IsUserCreated, + IsFromIndex = statistic.IsFromIndex, + HasFilter = statistic.HasFilter, + }, + cancellationToken); + } + } + + private async Task IsEnabledAsync(CancellationToken cancellationToken) + { + var value = await GetNumberParameterByIdAsync(IsEnabledId, cancellationToken); + return value == 1; + } + + private static TruncatedField Truncate(string value) + { + value ??= string.Empty; + var truncated = value.Length > MaxFieldLength; + return new TruncatedField(truncated ? value.Substring(0, MaxFieldLength) : value, truncated, value.Length); + } + + private sealed class SlowQueryResult + { + internal long QueryId { get; set; } + + internal long PlanId { get; set; } + + internal long ExecutionCount { get; set; } + + internal double TotalDurationMilliseconds { get; set; } + + internal double AverageDurationMilliseconds { get; set; } + + internal double MaxDurationMilliseconds { get; set; } + + internal double TotalCpuMilliseconds { get; set; } + + internal double AverageCpuMilliseconds { get; set; } + + internal double TotalLogicalReads { get; set; } + + internal double AverageLogicalReads { get; set; } + + internal string QueryText { get; set; } + + internal DateTimeOffset IntervalStart { get; set; } + + internal DateTimeOffset IntervalEnd { get; set; } + } + + private sealed class StatisticsHealthResult + { + internal string SchemaName { get; set; } + + internal string TableName { get; set; } + + internal string StatisticsName { get; set; } + + internal DateTimeOffset? LastUpdated { get; set; } + + internal long? Rows { get; set; } + + internal long? RowsSampled { get; set; } + + internal long? ModificationCounter { get; set; } + + internal double? ModificationPercent { get; set; } + + internal bool IsAutoCreated { get; set; } + + internal bool IsUserCreated { get; set; } + + internal bool IsFromIndex { get; set; } + + internal bool HasFilter { get; set; } + } + + private sealed class QueryStoreState + { + // sys.database_query_store_options.readonly_reason is int, not bigint. + internal QueryStoreState(string actualState, int? readonlyReason) + { + ActualState = actualState; + ReadonlyReason = readonlyReason; + } + + internal string ActualState { get; } + + internal int? ReadonlyReason { get; } + } + + private sealed class WaitStatistics + { + internal WaitStatistics(long planId, double totalWaitMilliseconds, string topWaitCategory) + { + PlanId = planId; + TotalWaitMilliseconds = totalWaitMilliseconds; + TopWaitCategory = topWaitCategory; + } + + internal long PlanId { get; } + + internal double TotalWaitMilliseconds { get; } + + internal string TopWaitCategory { get; } + } + + private sealed class QueryPlanResult + { + internal QueryPlanResult(long planId, string queryPlan) + { + PlanId = planId; + QueryPlan = queryPlan; + } + + internal long PlanId { get; } + + internal string QueryPlan { get; } + } + + private sealed class TruncatedField + { + internal TruncatedField(string value, bool truncated, int originalLength) + { + Value = value; + Truncated = truncated; + OriginalLength = originalLength; + } + + internal string Value { get; } + + internal bool Truncated { get; } + + internal int OriginalLength { get; } + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index 867c30cdb9..c349198ba3 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -29,6 +29,7 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; private readonly ExpiredResourceCleanupWatchdog _expiredResourceCleanupWatchdog; private readonly GeoReplicationLagWatchdog _geoReplicationLagWatchdog; + private readonly QueryStoreDiagnosticsWatchdog _queryStoreDiagnosticsWatchdog; private readonly JobMonitorWatchdog _jobMonitorWatchdog; private readonly CoreFeatureConfiguration _coreFeatureConfiguration; private readonly WatchdogConfiguration _watchdogConfiguration; @@ -40,6 +41,7 @@ public WatchdogsBackgroundService( InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog, ExpiredResourceCleanupWatchdog expiredResourceCleanupWatchdog, GeoReplicationLagWatchdog geoReplicationLagWatchdog, + QueryStoreDiagnosticsWatchdog queryStoreDiagnosticsWatchdog, JobMonitorWatchdog jobMonitorWatchdog, IOptions coreFeatureConfiguration, IOptions watchdogConfiguration) @@ -50,6 +52,7 @@ public WatchdogsBackgroundService( _invisibleHistoryCleanupWatchdog = EnsureArg.IsNotNull(invisibleHistoryCleanupWatchdog, nameof(invisibleHistoryCleanupWatchdog)); _expiredResourceCleanupWatchdog = EnsureArg.IsNotNull(expiredResourceCleanupWatchdog, nameof(expiredResourceCleanupWatchdog)); _geoReplicationLagWatchdog = geoReplicationLagWatchdog; // Can be null when feature is disabled + _queryStoreDiagnosticsWatchdog = EnsureArg.IsNotNull(queryStoreDiagnosticsWatchdog, nameof(queryStoreDiagnosticsWatchdog)); _jobMonitorWatchdog = EnsureArg.IsNotNull(jobMonitorWatchdog, nameof(jobMonitorWatchdog)); _coreFeatureConfiguration = EnsureArg.IsNotNull(coreFeatureConfiguration?.Value, nameof(coreFeatureConfiguration)); _watchdogConfiguration = EnsureArg.IsNotNull(watchdogConfiguration?.Value, nameof(watchdogConfiguration)); @@ -90,6 +93,11 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) tasks.Add(_expiredResourceCleanupWatchdog.ExecuteAsync(continuationTokenSource.Token)); } + if (_watchdogConfiguration.QueryStoreDiagnostics.Enabled) + { + tasks.Add(_queryStoreDiagnosticsWatchdog.ExecuteAsync(continuationTokenSource.Token)); + } + await Task.WhenAny(tasks); if (!stoppingToken.IsCancellationRequested) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index 426273b34c..9c6dfd9a2e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -1,7 +1,7 @@  - 117 + 116 Features\Schema\Migrations\$(LatestSchemaVersion).sql LatestSchemaVersion-$(LatestSchemaVersion) @@ -46,7 +46,6 @@ - diff --git a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs index 12fe9334b3..4778fdb462 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs @@ -209,6 +209,7 @@ public static IFhirServerBuilder AddSqlServer(this IFhirServerBuilder fhirServer services.Add().Singleton().AsSelf(); services.Add().Singleton().AsSelf(); services.Add().Singleton().AsSelf(); + services.Add().Singleton().AsSelf(); services.Add().Singleton().AsSelf(); services.Add().Scoped().AsSelf(); services.AddFactory>(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems index b76140f915..f743ed567f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems @@ -28,7 +28,7 @@ - + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs new file mode 100644 index 0000000000..5826d17d36 --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs @@ -0,0 +1,279 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Data; +using System.Threading; +using System.Threading.Tasks; +using Medino; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Metrics; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Tests.Integration.Persistence +{ + [FhirStorageTestsFixtureArgumentSets(DataStore.SqlServer)] + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.DataSourceValidation)] + public class QueryStoreDiagnosticsWatchdogTests : IClassFixture + { + private const int QueryExecutionCount = 2; + private const int QueryStorePollAttempts = 15; + private static readonly TimeSpan QueryStorePollInterval = TimeSpan.FromSeconds(1); + private readonly SqlServerFhirStorageTestsFixture _fixture; + + public QueryStoreDiagnosticsWatchdogTests(SqlServerFhirStorageTestsFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task GivenEnabledWatchdogAndCapturedProbe_WhenRun_ThenPublishesSlowQuerySanitizedPlanAndStatisticsHealth() + { + // Arrange + var mediator = Substitute.For(); + var notifications = new List(); + CaptureNotifications(mediator, notifications); + var watchdog = CreateWatchdog(mediator, enabled: true); + string tableName = $"DiagProbe_{Guid.NewGuid():N}"; + string queryAlias = $"probe_{Guid.NewGuid():N}"; + + await using SqlConnection connection = await _fixture.SqlConnectionBuilder.GetSqlConnectionAsync(cancellationToken: CancellationToken.None); + await connection.OpenAsync(CancellationToken.None); + + try + { + await EnableAndVerifyQueryStoreAsync(connection, CancellationToken.None); + await SetWatchdogParametersAsync(connection, isEnabled: 1, periodSeconds: 300, CancellationToken.None); + await CreateProbeTableAsync(connection, tableName, CancellationToken.None); + await ExecuteProbeQueryAsync(connection, tableName, queryAlias, CancellationToken.None); + await WaitForQueryStoreCaptureAsync(connection, queryAlias, CancellationToken.None); + + // Act + await watchdog.RunWorkForTestingAsync(CancellationToken.None); + + // Assert + SlowQueryNotification slowQuery = Assert.Single( + notifications.FindAll(notification => notification is SlowQueryNotification) + .ConvertAll(notification => (SlowQueryNotification)notification) + .FindAll(notification => notification.QueryText.Contains(queryAlias, StringComparison.Ordinal))); + Assert.True(slowQuery.QueryId > 0); + Assert.True(slowQuery.PlanId > 0); + Assert.True(slowQuery.QueryTextLength > 0); + + QueryPlanNotification queryPlan = Assert.Single( + notifications.FindAll(notification => notification is QueryPlanNotification) + .ConvertAll(notification => (QueryPlanNotification)notification) + .FindAll(notification => notification.QueryId == slowQuery.QueryId && notification.PlanId == slowQuery.PlanId)); + Assert.Equal(QueryPlanSanitizer.SanitizedStatus, queryPlan.SanitizationStatus); + Assert.NotNull(queryPlan.SanitizedQueryPlan); + + Assert.NotEmpty(notifications.FindAll(notification => notification is StatisticsHealthNotification)); + + foreach (IMetricsNotification notification in notifications.FindAll(notification => notification is SlowQueryNotification)) + { + string queryText = ((SlowQueryNotification)notification).QueryText; + Assert.DoesNotContain("query_store", queryText, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("dm_db_stats_properties", queryText, StringComparison.OrdinalIgnoreCase); + } + } + finally + { + await DropProbeTableAsync(connection, tableName, CancellationToken.None); + await DeleteWatchdogParametersAsync(connection, CancellationToken.None); + } + } + + [Fact] + public async Task GivenConfigurationGateDisabled_WhenRun_ThenPublishesNothing() + { + // Arrange + var mediator = Substitute.For(); + var watchdog = CreateWatchdog(mediator, enabled: false); + + // Act + await watchdog.RunWorkForTestingAsync(CancellationToken.None); + + // Assert + Assert.Empty(mediator.ReceivedCalls()); + } + + [Fact] + public async Task GivenRuntimeGateDisabled_WhenRun_ThenPublishesNothing() + { + // Arrange + var mediator = Substitute.For(); + var watchdog = CreateWatchdog(mediator, enabled: true); + await using SqlConnection connection = await _fixture.SqlConnectionBuilder.GetSqlConnectionAsync(cancellationToken: CancellationToken.None); + await connection.OpenAsync(CancellationToken.None); + await SetWatchdogParametersAsync(connection, isEnabled: 0, periodSeconds: 300, CancellationToken.None); + + try + { + // Act + await watchdog.RunWorkForTestingAsync(CancellationToken.None); + + // Assert + Assert.Empty(mediator.ReceivedCalls()); + } + finally + { + await DeleteWatchdogParametersAsync(connection, CancellationToken.None); + } + } + + private QueryStoreDiagnosticsWatchdog CreateWatchdog(IMediator mediator, bool enabled) + { + var configuration = new WatchdogConfiguration(); + configuration.QueryStoreDiagnostics.Enabled = enabled; + configuration.QueryStoreDiagnostics.PeriodSec = 300; + configuration.QueryStoreDiagnostics.SlowQueryCount = 100; + configuration.QueryStoreDiagnostics.MinDurationMilliseconds = 1; + configuration.QueryStoreDiagnostics.IncludeQueryPlans = true; + configuration.QueryStoreDiagnostics.IncludeStatisticsHealth = true; + configuration.QueryStoreDiagnostics.StatisticsHealthCount = 50; + + return new QueryStoreDiagnosticsWatchdog( + _fixture.SqlRetryService, + NullLogger.Instance, + mediator, + Options.Create(configuration)); + } + + private static void CaptureNotifications(IMediator mediator, List notifications) + { + mediator.When(x => x.PublishAsync(Arg.Any(), Arg.Any())) + .Do(info => notifications.Add((SlowQueryNotification)info[0])); + mediator.When(x => x.PublishAsync(Arg.Any(), Arg.Any())) + .Do(info => notifications.Add((QueryPlanNotification)info[0])); + mediator.When(x => x.PublishAsync(Arg.Any(), Arg.Any())) + .Do(info => notifications.Add((StatisticsHealthNotification)info[0])); + } + + private static async Task EnableAndVerifyQueryStoreAsync(SqlConnection connection, CancellationToken cancellationToken) + { + string initialState = await GetQueryStoreStateAsync(connection, cancellationToken); + if (!string.Equals(initialState, "READ_WRITE", StringComparison.OrdinalIgnoreCase)) + { + await ExecuteNonQueryAsync(connection, "ALTER DATABASE CURRENT SET QUERY_STORE = ON;", cancellationToken); + } + + await ExecuteNonQueryAsync( + connection, + "ALTER DATABASE CURRENT SET QUERY_STORE (OPERATION_MODE = READ_WRITE, QUERY_CAPTURE_MODE = ALL);", + cancellationToken); + + Assert.Equal("READ_WRITE", await GetQueryStoreStateAsync(connection, cancellationToken), ignoreCase: true); + } + + private static async Task GetQueryStoreStateAsync(SqlConnection connection, CancellationToken cancellationToken) + { + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT actual_state_desc FROM sys.database_query_store_options;"; + return (string)await command.ExecuteScalarAsync(cancellationToken); + } + + private static async Task SetWatchdogParametersAsync(SqlConnection connection, int isEnabled, int periodSeconds, CancellationToken cancellationToken) + { + await SetParameterAsync(connection, "QueryStoreDiagnosticsWatchdog.IsEnabled", isEnabled, cancellationToken); + await SetParameterAsync(connection, "QueryStoreDiagnosticsWatchdog.PeriodSec", periodSeconds, cancellationToken); + } + + private static async Task SetParameterAsync(SqlConnection connection, string id, double value, CancellationToken cancellationToken) + { + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = @" +UPDATE dbo.Parameters SET Number = @Value WHERE Id = @Id; +IF @@ROWCOUNT = 0 +BEGIN + INSERT INTO dbo.Parameters (Id, Number) VALUES (@Id, @Value); +END"; + command.Parameters.AddWithValue("@Id", id); + command.Parameters.AddWithValue("@Value", value); + await command.ExecuteNonQueryAsync(cancellationToken); + } + + private static async Task DeleteWatchdogParametersAsync(SqlConnection connection, CancellationToken cancellationToken) + { + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = @" +DELETE FROM dbo.Parameters +WHERE Id IN ('QueryStoreDiagnosticsWatchdog.IsEnabled', 'QueryStoreDiagnosticsWatchdog.PeriodSec');"; + await command.ExecuteNonQueryAsync(cancellationToken); + } + + private static async Task CreateProbeTableAsync(SqlConnection connection, string tableName, CancellationToken cancellationToken) + { + await ExecuteNonQueryAsync( + connection, + $"CREATE TABLE dbo.[{tableName}] (Id int NOT NULL PRIMARY KEY); INSERT INTO dbo.[{tableName}] (Id) SELECT TOP (200) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) FROM sys.all_objects;", + cancellationToken); + } + + private static async Task DropProbeTableAsync(SqlConnection connection, string tableName, CancellationToken cancellationToken) + { + await ExecuteNonQueryAsync(connection, $"DROP TABLE IF EXISTS dbo.[{tableName}];", cancellationToken); + } + + private static async Task ExecuteProbeQueryAsync(SqlConnection connection, string tableName, string queryAlias, CancellationToken cancellationToken) + { + for (int execution = 0; execution < QueryExecutionCount; execution++) + { + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = $"SELECT SUM(CONVERT(bigint, firstProbe.Id) * secondProbe.Id * thirdProbe.Id) AS [{queryAlias}] FROM dbo.[{tableName}] AS firstProbe CROSS JOIN dbo.[{tableName}] AS secondProbe CROSS JOIN dbo.[{tableName}] AS thirdProbe;"; + object result = await command.ExecuteScalarAsync(cancellationToken); + Assert.NotNull(result); + } + } + + private static async Task WaitForQueryStoreCaptureAsync(SqlConnection connection, string queryAlias, CancellationToken cancellationToken) + { + for (int attempt = 0; attempt < QueryStorePollAttempts; attempt++) + { + await ExecuteNonQueryAsync(connection, "EXEC sys.sp_query_store_flush_db;", cancellationToken); + + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = @" +SELECT COUNT_BIG(*) +FROM sys.query_store_runtime_stats AS runtimeStats +INNER JOIN sys.query_store_plan AS queryPlan + ON runtimeStats.plan_id = queryPlan.plan_id +INNER JOIN sys.query_store_query AS queryStoreQuery + ON queryPlan.query_id = queryStoreQuery.query_id +INNER JOIN sys.query_store_query_text AS queryText + ON queryStoreQuery.query_text_id = queryText.query_text_id +WHERE queryText.query_sql_text LIKE @QueryTextPattern + AND runtimeStats.execution_type = 0;"; + command.Parameters.Add("@QueryTextPattern", SqlDbType.NVarChar, 256).Value = $"%{queryAlias}%"; + + long capturedRuntimeStatisticsCount = Convert.ToInt64(await command.ExecuteScalarAsync(cancellationToken)); + if (capturedRuntimeStatisticsCount > 0) + { + return; + } + + await Task.Delay(QueryStorePollInterval, cancellationToken); + } + + Assert.Fail("Query Store did not persist regular runtime statistics for the GUID-alias probe query after the supported flush and polling window."); + } + + private static async Task ExecuteNonQueryAsync(SqlConnection connection, string commandText, CancellationToken cancellationToken) + { + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = commandText; + await command.ExecuteNonQueryAsync(cancellationToken); + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs deleted file mode 100644 index 884a40956b..0000000000 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerQueryStoreDiagnosticsTests.cs +++ /dev/null @@ -1,264 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Data; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Data.SqlClient; -using Microsoft.Health.Fhir.Core.Features.Persistence; -using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using Microsoft.Health.Test.Utilities; -using Xunit; - -namespace Microsoft.Health.Fhir.Tests.Integration.Persistence -{ - /// - /// Exercises the SQL Server Query Store diagnostic procedures against captured FHIR database activity. - /// - [FhirStorageTestsFixtureArgumentSets(DataStore.SqlServer)] - [Trait(Traits.OwningTeam, OwningTeam.Fhir)] - [Trait(Traits.Category, Categories.DataSourceValidation)] - public class SqlServerQueryStoreDiagnosticsTests : IClassFixture - { - private const int QueryExecutionCount = 4; - private const int QueryStorePollAttempts = 15; - private static readonly TimeSpan QueryStorePollInterval = TimeSpan.FromSeconds(1); - private readonly FhirStorageTestsFixture _fixture; - - /// - /// Initializes a new instance of the class. - /// - /// The SQL Server-backed FHIR storage fixture. - public SqlServerQueryStoreDiagnosticsTests(FhirStorageTestsFixture fixture) - { - _fixture = fixture; - } - - /// - /// Verifies the slow-query, plan diagnostics, and statistics health contracts with Query Store runtime data. - /// - /// A task that represents the asynchronous test operation. - [Fact] - public async Task GivenAQueryStoreCapturedFhirQuery_WhenDiagnosticsProceduresAreCalled_ThenReturnSanitizedPlanAndStatisticsMetadata() - { - using SqlConnection connection = await _fixture.SqlHelper.GetSqlConnectionAsync(); - if (connection.State != ConnectionState.Open) - { - await connection.OpenAsync(CancellationToken.None); - } - - await EnableAndVerifyQueryStoreAsync(connection, CancellationToken.None); - - string queryMarker = $"QueryStoreDiagnostics{Guid.NewGuid():N}"; - DateTimeOffset windowStart = DateTimeOffset.UtcNow.AddMinutes(-5); - for (int execution = 0; execution < QueryExecutionCount; execution++) - { - using SqlCommand command = connection.CreateCommand(); - command.CommandText = $"SELECT COUNT_BIG(*) AS [{queryMarker}] FROM dbo.Resource;"; - object result = await command.ExecuteScalarAsync(CancellationToken.None); - Assert.NotNull(result); - } - - await WaitForQueryStoreCaptureAsync(connection, queryMarker, CancellationToken.None); - - long planId = await GetSlowQueryPlanIdAsync( - connection, - queryMarker, - windowStart, - DateTimeOffset.UtcNow.AddMinutes(1), - CancellationToken.None); - - await AssertPlanDiagnosticsAsync(connection, planId, queryMarker, CancellationToken.None); - await AssertResourceStatisticsHealthAsync(connection, CancellationToken.None); - } - - private static async Task EnableAndVerifyQueryStoreAsync(SqlConnection connection, CancellationToken cancellationToken) - { - string initialState = await GetQueryStoreStateAsync(connection, cancellationToken); - - if (!string.Equals(initialState, "READ_WRITE", StringComparison.OrdinalIgnoreCase)) - { - await ExecuteNonQueryAsync( - connection, - "ALTER DATABASE CURRENT SET QUERY_STORE = ON;", - cancellationToken); - } - - await ExecuteNonQueryAsync( - connection, - "ALTER DATABASE CURRENT SET QUERY_STORE (OPERATION_MODE = READ_WRITE, QUERY_CAPTURE_MODE = ALL);", - cancellationToken); - - string finalState = await GetQueryStoreStateAsync(connection, cancellationToken); - Assert.True( - string.Equals(finalState, "READ_WRITE", StringComparison.OrdinalIgnoreCase), - $"Query Store is not writable after enablement (state: {finalState ?? "unknown"})."); - } - - private static async Task GetQueryStoreStateAsync(SqlConnection connection, CancellationToken cancellationToken) - { - using SqlCommand command = connection.CreateCommand(); - command.CommandText = "SELECT actual_state_desc FROM sys.database_query_store_options;"; - return (string)await command.ExecuteScalarAsync(cancellationToken); - } - - private static async Task WaitForQueryStoreCaptureAsync(SqlConnection connection, string queryMarker, CancellationToken cancellationToken) - { - for (int attempt = 0; attempt < QueryStorePollAttempts; attempt++) - { - await ExecuteNonQueryAsync(connection, "EXEC sys.sp_query_store_flush_db;", cancellationToken); - - using SqlCommand command = connection.CreateCommand(); - command.CommandText = """ - SELECT COUNT_BIG(*) - FROM sys.query_store_query_text - WHERE query_sql_text LIKE @queryTextPattern; - """; - command.Parameters.Add("@queryTextPattern", SqlDbType.NVarChar, 256).Value = $"%{queryMarker}%"; - - long capturedQueryCount = Convert.ToInt64(await command.ExecuteScalarAsync(cancellationToken)); - if (capturedQueryCount > 0) - { - return; - } - - await Task.Delay(QueryStorePollInterval, cancellationToken); - } - - Assert.Fail("Query Store did not persist the diagnostic query after the supported flush and polling window."); - } - - private static async Task GetSlowQueryPlanIdAsync( - SqlConnection connection, - string queryMarker, - DateTimeOffset windowStart, - DateTimeOffset windowEnd, - CancellationToken cancellationToken) - { - using SqlCommand command = connection.CreateCommand(); - command.CommandType = CommandType.StoredProcedure; - command.CommandText = "dbo.GetQueryStoreSlowQueries"; - command.Parameters.Add("@StartTime", SqlDbType.DateTimeOffset).Value = windowStart; - command.Parameters.Add("@EndTime", SqlDbType.DateTimeOffset).Value = windowEnd; - command.Parameters.Add("@Top", SqlDbType.Int).Value = 10; - command.Parameters.Add("@Offset", SqlDbType.Int).Value = 0; - command.Parameters.Add("@OrderBy", SqlDbType.VarChar, 32).Value = "TotalWait"; - command.Parameters.Add("@MinExecutions", SqlDbType.BigInt).Value = QueryExecutionCount; - command.Parameters.Add("@QueryTextContains", SqlDbType.NVarChar, 256).Value = queryMarker; - - using SqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken); - Assert.True(await reader.ReadAsync(cancellationToken), "The uniquely marked Query Store query was not returned by dbo.GetQueryStoreSlowQueries."); - - long planId = reader.GetInt64(reader.GetOrdinal("PlanId")); - Assert.True(planId > 0); - Assert.True(reader.GetInt64(reader.GetOrdinal("QueryId")) > 0); - Assert.True(reader.GetInt64(reader.GetOrdinal("RegularExecutionCount")) >= QueryExecutionCount); - Assert.Contains(queryMarker, reader.GetString(reader.GetOrdinal("QuerySqlText"))); - Assert.False(reader.IsDBNull(reader.GetOrdinal("FirstExecutionTimeUtc"))); - Assert.False(reader.IsDBNull(reader.GetOrdinal("LastExecutionTimeUtc"))); - AssertWaitStatisticsAvailability(reader); - Assert.False(await reader.ReadAsync(cancellationToken), "The unique query-text filter returned more than one Query Store plan."); - Assert.False(await reader.NextResultAsync(cancellationToken), "dbo.GetQueryStoreSlowQueries returned more than one result set."); - - return planId; - } - - private static void AssertWaitStatisticsAvailability(SqlDataReader reader) - { - int totalWaitMillisecondsOrdinal = reader.GetOrdinal("TotalWaitMilliseconds"); - int averageWaitMillisecondsOrdinal = reader.GetOrdinal("AverageWaitMilliseconds"); - int waitStatsStatusOrdinal = reader.GetOrdinal("WaitStatsStatus"); - int waitStatsXmlOrdinal = reader.GetOrdinal("WaitStatsXml"); - string waitStatsStatus = reader.GetString(waitStatsStatusOrdinal); - - Assert.Contains(waitStatsStatus, new[] { "Available", "Disabled", "Unavailable" }); - - if (string.Equals(waitStatsStatus, "Available", StringComparison.Ordinal)) - { - Assert.False(reader.IsDBNull(totalWaitMillisecondsOrdinal)); - Assert.False(reader.IsDBNull(averageWaitMillisecondsOrdinal)); - Assert.False(reader.IsDBNull(waitStatsXmlOrdinal)); - } - else - { - Assert.True(reader.IsDBNull(totalWaitMillisecondsOrdinal)); - Assert.True(reader.IsDBNull(averageWaitMillisecondsOrdinal)); - Assert.True(reader.IsDBNull(waitStatsXmlOrdinal)); - } - } - - private static async Task AssertPlanDiagnosticsAsync( - SqlConnection connection, - long planId, - string queryMarker, - CancellationToken cancellationToken) - { - using SqlCommand command = connection.CreateCommand(); - command.CommandType = CommandType.StoredProcedure; - command.CommandText = "dbo.GetQueryStorePlanDiagnostics"; - command.Parameters.Add("@PlanId", SqlDbType.BigInt).Value = planId; - - using SqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken); - Assert.True(await reader.ReadAsync(cancellationToken), "dbo.GetQueryStorePlanDiagnostics did not return the selected plan."); - Assert.Equal(planId, reader.GetInt64(reader.GetOrdinal("PlanId"))); - Assert.True(reader.GetInt64(reader.GetOrdinal("QueryId")) > 0); - Assert.Contains(queryMarker, reader.GetString(reader.GetOrdinal("QuerySqlText"))); - Assert.False(reader.IsDBNull(reader.GetOrdinal("CompatibilityLevel"))); - - string sanitizationStatus = reader.GetString(reader.GetOrdinal("SanitizationStatus")); - int sanitizedShowPlanXmlOrdinal = reader.GetOrdinal("SanitizedShowPlanXml"); - - if (string.Equals(sanitizationStatus, "Sanitized", StringComparison.Ordinal)) - { - Assert.False(reader.IsDBNull(sanitizedShowPlanXmlOrdinal)); - - string sanitizedShowPlanXml = reader.GetValue(sanitizedShowPlanXmlOrdinal).ToString()!; - Assert.False(sanitizedShowPlanXml.Contains("ParameterList", StringComparison.OrdinalIgnoreCase)); - Assert.False(sanitizedShowPlanXml.Contains("ParameterCompiledValue", StringComparison.OrdinalIgnoreCase)); - Assert.False(sanitizedShowPlanXml.Contains("ParameterRuntimeValue", StringComparison.OrdinalIgnoreCase)); - } - else - { - Assert.Contains(sanitizationStatus, new[] { "PlanXmlUnavailable", "InvalidXml", "VerificationFailed" }); - Assert.True(reader.IsDBNull(sanitizedShowPlanXmlOrdinal)); - Assert.False(reader.IsDBNull(reader.GetOrdinal("SanitizationErrorCode"))); - } - - Assert.False(await reader.ReadAsync(cancellationToken), "dbo.GetQueryStorePlanDiagnostics returned more than one row."); - Assert.False(await reader.NextResultAsync(cancellationToken), "dbo.GetQueryStorePlanDiagnostics returned more than one result set."); - } - - private static async Task AssertResourceStatisticsHealthAsync(SqlConnection connection, CancellationToken cancellationToken) - { - using SqlCommand command = connection.CreateCommand(); - command.CommandType = CommandType.StoredProcedure; - command.CommandText = "dbo.GetStatisticsHealth"; - command.Parameters.Add("@TableName", SqlDbType.NVarChar, 128).Value = "Resource"; - command.Parameters.Add("@Top", SqlDbType.Int).Value = 1; - command.Parameters.Add("@Offset", SqlDbType.Int).Value = 0; - command.Parameters.Add("@OrderBy", SqlDbType.VarChar, 32).Value = "Rows"; - - using SqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken); - Assert.True(await reader.ReadAsync(cancellationToken), "dbo.GetStatisticsHealth did not return metadata for dbo.Resource."); - Assert.Equal("Resource", reader.GetString(reader.GetOrdinal("TableName"))); - Assert.False(reader.IsDBNull(reader.GetOrdinal("StatisticsName"))); - Assert.True(reader.GetInt32(reader.GetOrdinal("StatisticsId")) > 0); - Assert.False(reader.IsDBNull(reader.GetOrdinal("StatisticsColumns"))); - Assert.Contains("StatisticsColumn", reader.GetValue(reader.GetOrdinal("StatisticsColumns")).ToString()); - Assert.Contains(reader.GetString(reader.GetOrdinal("StatisticsStatus")), new[] { "Available", "PropertiesUnavailable" }); - Assert.False(await reader.ReadAsync(cancellationToken), "The bounded Resource statistics request returned more than one row."); - Assert.False(await reader.NextResultAsync(cancellationToken), "dbo.GetStatisticsHealth returned more than one result set."); - } - - private static async Task ExecuteNonQueryAsync(SqlConnection connection, string commandText, CancellationToken cancellationToken) - { - using SqlCommand command = connection.CreateCommand(); - command.CommandText = commandText; - await command.ExecuteNonQueryAsync(cancellationToken); - } - } -} From 745675da78e2279b74a47f21c66b985b467ec522 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 20:00:36 +0000 Subject: [PATCH 10/20] Address PR review findings for Query Store diagnostics watchdog Bind every diagnostics read to the primary. The reads passed isReadOnly: true, which lets ReplicaHandler route them to a read-only secondary. Query Store reports READ_ONLY there, so the desired_state gate returned early and the feature would silently emit nothing. Make failure observable rather than silent: - the missing-Query-Store path (error 208) logs Warning, not Debug - wait statistics failure is surfaced on the notification through a new WaitStatisticsStatus property instead of being flattened into zeros - each completed tick logs its window and counts, including zeros, so a watchdog that runs but collects nothing is distinguishable from one that is not running at all - sanitization failure logs Warning with the plan id and status - readonly_reason is decoded from its bitmask into text Harden the PHI boundary. Post-sanitization verification walked the serialized string, so any plan whose own StatementText contained the literal "ParameterList" was discarded as a false positive. It now walks the element tree by local name, which is both accurate and namespace- agnostic. Make QueryPlanSanitizationResult unable to hold a contradictory state: private constructor, four named factories, and Truncated derived from the payload rather than passed alongside it. Also warn when configuration disables collection, document the WHY behind each SQL trap, and correct the failure-containment claim in the design doc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 35 ++- .../Features/Metrics/SlowQueryNotification.cs | 9 + .../QueryPlanSanitizationResultTests.cs | 108 +++++++++ .../Watchdogs/QueryPlanSanitizerTests.cs | 25 +- .../QueryStoreReadonlyReasonTests.cs | 81 +++++++ .../Watchdogs/QueryPlanSanitizationResult.cs | 79 ++++++- .../Features/Watchdogs/QueryPlanSanitizer.cs | 45 +++- .../QueryStoreDiagnosticsWatchdog.cs | 217 +++++++++++++++--- .../QueryStoreDiagnosticsWatchdogTests.cs | 67 +++++- 9 files changed, 603 insertions(+), 63 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizationResultTests.cs create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 2e4da90741..473267072f 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -31,7 +31,7 @@ A **watchdog** — the repository's existing leased background-worker pattern ```text FHIR server instance (lease holder) └── QueryStoreDiagnosticsWatchdog every PeriodSec, default 3600s - ├── sys.database_query_store_options state check, read-only + ├── sys.database_query_store_options state gate, always primary ├── sys.query_store_* slow plans + query text ├── sys.query_store_wait_stats best effort ├── sys.stats / sys.dm_db_stats_properties @@ -85,7 +85,9 @@ Three notification types implement `IMetricsNotification`, each reporting `FhirO ### `SlowQueryNotification` -One per slow plan per tick. Carries `QueryId`, `PlanId`, execution count, total/average/maximum duration, total/average CPU, total/average logical reads, total/average wait time, top wait category, the Query Store query text, and the collection window bounds. +One per slow plan per tick. Carries `QueryId`, `PlanId`, execution count, total/average/maximum duration, total/average CPU, total/average logical reads, total/average wait time, top wait category, `WaitStatisticsStatus`, the Query Store query text, and the collection window bounds. + +`WaitStatisticsStatus` describes why the wait fields look the way they do: `Available` when wait statistics were read for the plan, `Unavailable` when the wait query succeeded but returned no row for it (wait capture off, or no waits accrued), and `Failed` when the wait query itself threw. Without it, "wait capture is switched off" and "the wait query has been failing for a month" are indistinguishable, because both leave the wait fields null. `QueryId` and `PlanId` are the join keys back into Query Store, and into the `QueryStoreRuntimeStatistics` stream already exported to Log Analytics, so a responder can correlate an emitted slow query with the existing diagnostic-settings telemetry. @@ -129,7 +131,9 @@ The watchdog's own Query Store reads are excluded by filtering out query text re ### Wait statistics -Wait statistics are collected by a **separate, best-effort** query and merged in C#. A failure — most commonly `sys.query_store_wait_stats` being unavailable, or wait capture being off — is logged at debug level and leaves the wait fields null. Runtime results are still emitted. +Wait statistics are collected by a **separate, best-effort** query and merged in C#. A failure — most commonly `sys.query_store_wait_stats` being unavailable, or wait capture being off — is logged as a warning, leaves the wait fields null, and sets `WaitStatisticsStatus` to `Failed`. Runtime results are still emitted. + +`SqlException` is caught broadly there on purpose, so that a transient wait-query failure cannot abort the tick and suppress the runtime metrics. The trade-off accepted is that timeouts, deadlocks, permission denials and missing views are not distinguished from one another at that point; the warning carries the exception, and the notification carries the status. Wait capture is retained locally, rather than deferred entirely to the `QueryStoreWaitStatistics` Log Analytics stream, so that a single emitted slow-query record is self-contained: runtime metrics, waits, query text, and plan identity arrive together without a join against a second telemetry source. @@ -143,6 +147,22 @@ Modification percentage is left null when the row count is null or zero rather t `WatchdogsBackgroundService` cancels **every** watchdog if any one of them fails. A diagnostic feature must never be able to take down transaction or cleanup watchdogs, so the collection body contains its own failures: missing views and permission denials are logged and the tick returns rather than propagating. +That containment covers **per-tick collection only**. `Watchdog.ExecuteAsync` awaits `InitParamsAsync` *before* and *outside* `FhirTimer`'s per-tick catch, so a throw during initialization — seeding the `dbo.Parameters` rows — still faults the watchdog task, and `WatchdogsBackgroundService` cancels the rest. This is a **pre-existing property of the shared watchdog framework**, not something this feature introduces: `DefragWatchdog` initializes with the identical insert pattern. It is documented here rather than worked around, because changing the shared framework is out of scope for a diagnostics feature. + +### Reading the primary + +Every diagnostics read binds to the **primary**, not to a read-only replica, even though these are all read-only queries. + +Query Store state is primary-scoped: on a secondary the database is read-only, so `sys.database_query_store_options.actual_state_desc` reports `READ_ONLY`, the `READ_WRITE` state gate rejects it, and the tick returns having emitted nothing — silently, forever, with both gates on and no exception raised. Replica routing is also decided per call against a process-global counter, so a single tick could otherwise check state on the primary and read data from a secondary, harvesting plan identifiers on one server and looking them up on another. + +The cost is one collection per period against the primary — hourly by default — which is negligible next to the failure mode it removes. + +### Configuration that disables collection + +A non-positive `SlowQueryCount` or `StatisticsHealthCount` reduces the corresponding query to `TOP (0)` or skips the section entirely, which is indistinguishable from a healthy empty result. Both cases are logged as a warning once per tick, and a section turned off deliberately through `IncludeQueryPlans` or `IncludeStatisticsHealth` is logged at information level. Misconfiguration is never fatal: a diagnostics feature must not fail the host. + +A completed tick logs one information-level line carrying the collection window and the counts of slow queries, plans and statistics rows published, **including zeros**, so that "the watchdog has been dead for three days" is distinguishable from "there were no slow queries". + ## Sanitization Showplan sanitization is performed **in C#** by `QueryPlanSanitizer`, not in T-SQL. The previous design did this with 236 lines of XML DML inside a stored procedure; the C# implementation is materially more reliable and easier to test with a fixture corpus. @@ -152,10 +172,12 @@ The sanitizer: 1. parses with an `XmlReader` configured with `DtdProcessing.Prohibit` and a null resolver, so a hostile or malformed plan cannot trigger entity resolution; 2. removes every `ParameterList` element and every `ParameterCompiledValue` and `ParameterRuntimeValue` attribute, matching on local name so that Showplan namespace differences between SQL versions cannot cause a miss; 3. re-serializes without formatting; -4. **verifies** that none of those three names survive anywhere in the serialized output, and returns `VerificationFailed` with null XML if any do; and +4. **verifies** structurally — by re-walking the parsed tree after removal and checking element and attribute local names, not by scanning the serialized text — that none of those three names survive, and returns `VerificationFailed` with null XML if any do; and 5. only then truncates to the field cap. -Step 4 is defence in depth: the plan is never emitted on the strength of the removal logic alone. +Step 4 is defence in depth: the plan is never emitted on the strength of the removal logic alone. It is structural because Showplan embeds the original SQL in `StatementText`, so a text scan would drop — silently and permanently — any plan whose own query text happens to contain the literal string `ParameterList`. + +`QueryPlanSanitizationResult` is constructed only through static factories, so "not verified but populated" is unconstructable rather than merely unused: every failure factory forces the XML to null, the success factory refuses a null document, and the truncation flag is derived from the payload rather than supplied alongside it. ### Disclosure boundary @@ -243,8 +265,9 @@ Secondary benefits of the change: 3. `StatisticsHealthNotification` rows are emitted for user tables. 4. The watchdog performs no work when either gate is off. 5. A non-`READ_WRITE` Query Store state is handled without error and without emission. -6. Wait-statistic unavailability degrades to null wait fields while runtime results are still emitted. +6. Wait-statistic unavailability degrades to null wait fields while runtime results are still emitted, and `WaitStatisticsStatus` reports which of the three outcomes occurred. 7. The watchdog does not report its own Query Store queries. +8. Emitted content is asserted, not merely emission: the execution count matches the number of probe executions, and duration and CPU sit in a plausible millisecond range, which is what catches a regression in the weighted rollup or in the microsecond-to-millisecond conversion. ## References diff --git a/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs b/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs index d254f79cfe..4ca0263089 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs @@ -77,6 +77,15 @@ public class SlowQueryNotification : IMetricsNotification /// public string TopWaitCategory { get; set; } + /// + /// Gets or sets the outcome of wait-statistics collection, so that absent wait fields are self-describing: + /// Available when wait statistics were read for this plan, Unavailable when the wait query + /// succeeded but returned no row for this plan (typically wait capture is off, or the plan accrued no waits), + /// and Failed when the wait query itself threw. Failed means the wait fields are missing because + /// collection is broken, not because there was nothing to report. + /// + public string WaitStatisticsStatus { get; set; } + /// /// Gets or sets the query text, limited to the diagnostics field-length cap. /// diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizationResultTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizationResultTests.cs new file mode 100644 index 0000000000..b30ae560cd --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizationResultTests.cs @@ -0,0 +1,108 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using System.Reflection; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public class QueryPlanSanitizationResultTests + { + [Fact] + public void GivenTheResultType_WhenInspected_ThenOnlyFactoriesCanCreateIt() + { + // Act + ConstructorInfo[] constructors = typeof(QueryPlanSanitizationResult) + .GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + + // Assert + Assert.NotEmpty(constructors); + Assert.All(constructors, constructor => Assert.True(constructor.IsPrivate)); + } + + [Fact] + public void GivenSanitizedXmlWithinTheFieldCap_WhenCreated_ThenCarriesThePlanAndIsNotTruncated() + { + // Arrange + const string xml = ""; + + // Act + QueryPlanSanitizationResult result = QueryPlanSanitizationResult.Sanitized(xml, originalLength: 64, sanitizedLength: xml.Length); + + // Assert + Assert.Equal(QueryPlanSanitizer.SanitizedStatus, result.Status); + Assert.Equal(xml, result.Xml); + Assert.False(result.Truncated); + Assert.Equal(64, result.OriginalLength); + Assert.Equal(xml.Length, result.SanitizedLength); + } + + [Fact] + public void GivenSanitizedXmlShorterThanTheSanitizedLength_WhenCreated_ThenTruncationIsDerivedRatherThanDeclared() + { + // Arrange + const string xml = "(() => QueryPlanSanitizationResult.Sanitized(null, 10, 10)); + } + + [Fact] + public void GivenASanitizedLengthBelowThePayloadLength_WhenCreatingASanitizedResult_ThenTheIllegalCombinationIsRejected() + { + // Act + Assert + Assert.Throws(() => QueryPlanSanitizationResult.Sanitized("", 128, 1)); + } + + [Fact] + public void GivenNegativeLengths_WhenCreatingResults_ThenTheIllegalCombinationsAreRejected() + { + // Act + Assert + Assert.Throws(() => QueryPlanSanitizationResult.Sanitized("", -1, 15)); + Assert.Throws(() => QueryPlanSanitizationResult.InvalidXml(-1)); + Assert.Throws(() => QueryPlanSanitizationResult.VerificationFailed(-1, 0)); + Assert.Throws(() => QueryPlanSanitizationResult.VerificationFailed(0, -1)); + } + + [Fact] + public void GivenAnyFailureFactory_WhenCreated_ThenTheXmlIsAlwaysNullAndNeverTruncated() + { + // Act + QueryPlanSanitizationResult[] results = + { + QueryPlanSanitizationResult.PlanXmlUnavailable(), + QueryPlanSanitizationResult.InvalidXml(128), + QueryPlanSanitizationResult.VerificationFailed(128, 64), + }; + + // Assert + Assert.All(results, result => Assert.Null(result.Xml)); + Assert.All(results, result => Assert.False(result.Truncated)); + Assert.All(results, result => Assert.NotEqual(QueryPlanSanitizer.SanitizedStatus, result.Status)); + Assert.Equal( + new[] { QueryPlanSanitizer.PlanXmlUnavailableStatus, QueryPlanSanitizer.InvalidXmlStatus, QueryPlanSanitizer.VerificationFailedStatus }, + results.Select(result => result.Status)); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs index b154b76f53..fce7150953 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs @@ -198,13 +198,31 @@ public void GivenSanitizedPlanExceedingFieldCap_WhenSanitized_ThenReturnsVerifie } [Fact] - public void GivenPlanThatFailsVerification_WhenSanitized_ThenNeverReturnsRawXml() + public void GivenPlanWhoseStatementTextContainsTheLiteralParameterList_WhenSanitized_ThenStillSanitizesSuccessfully() { // Arrange - const string queryPlan = @""; + // Showplan embeds the original SQL in StatementText. A serialized-text scan would treat this plan as + // unverifiable and drop it silently; structural verification must not. + const string queryPlan = @""; // Act - QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, 1); + QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, QueryStoreDiagnosticsWatchdog.MaxFieldLength); + + // Assert + Assert.Equal(QueryPlanSanitizer.SanitizedStatus, result.Status); + Assert.NotNull(result.Xml); + Assert.False(result.Truncated); + Assert.Contains("FROM dbo.ParameterList", result.Xml, StringComparison.Ordinal); + } + + [Theory] + [InlineData(@"")] + [InlineData(@"")] + [InlineData(@"N'MRN-12345'")] + public void GivenSensitiveNameThatSurvivesRemoval_WhenSanitized_ThenFailsVerificationAndNeverReturnsXml(string queryPlan) + { + // Act + QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, QueryStoreDiagnosticsWatchdog.MaxFieldLength); // Assert Assert.Equal(QueryPlanSanitizer.VerificationFailedStatus, result.Status); @@ -212,7 +230,6 @@ public void GivenPlanThatFailsVerification_WhenSanitized_ThenNeverReturnsRawXml( Assert.False(result.Truncated); Assert.Equal(queryPlan.Length, result.OriginalLength); Assert.True(result.SanitizedLength > 0); - Assert.NotEqual(queryPlan, result.Xml); } private static void AssertNoParameterMetadata(string queryPlan) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs new file mode 100644 index 0000000000..bf53037995 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs @@ -0,0 +1,81 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public class QueryStoreReadonlyReasonTests + { + [Theory] + [InlineData(1, "database is in read-only mode")] + [InlineData(2, "database is in single-user mode")] + [InlineData(4, "database is in emergency mode")] + [InlineData(8, "database is a secondary replica")] + [InlineData(65536, "Query Store has reached its size limit (MAX_STORAGE_SIZE_MB)")] + [InlineData(131072, "Query Store has reached the limit on the number of statements")] + public void GivenASingleDocumentedBit_WhenDescribed_ThenReturnsThatReason(int readonlyReason, string expected) + { + // Act + string description = QueryStoreDiagnosticsWatchdog.DescribeReadonlyReason(readonlyReason); + + // Assert + Assert.Equal(expected, description); + } + + [Fact] + public void GivenACombinedMask_WhenDescribed_ThenReturnsEveryReasonInBitOrder() + { + // Arrange + const int combinedMask = 8 | 65536 | 131072; + + // Act + string description = QueryStoreDiagnosticsWatchdog.DescribeReadonlyReason(combinedMask); + + // Assert + Assert.Equal( + "database is a secondary replica, Query Store has reached its size limit (MAX_STORAGE_SIZE_MB), Query Store has reached the limit on the number of statements", + description); + } + + [Fact] + public void GivenNoReason_WhenDescribed_ThenDistinguishesNotReportedFromNone() + { + // Act + string notReported = QueryStoreDiagnosticsWatchdog.DescribeReadonlyReason(null); + string none = QueryStoreDiagnosticsWatchdog.DescribeReadonlyReason(0); + + // Assert + Assert.Equal("not reported", notReported); + Assert.Equal("none", none); + } + + [Fact] + public void GivenAnUndocumentedBit_WhenDescribed_ThenReportsItAsUnrecognizedRatherThanEmpty() + { + // Act + string description = QueryStoreDiagnosticsWatchdog.DescribeReadonlyReason(1 << 20); + + // Assert + Assert.Equal("unrecognized reason", description); + } + + [Fact] + public void GivenTheSizeLimitBitCombinedWithAnUndocumentedBit_WhenDescribed_ThenStillNamesTheDocumentedReason() + { + // Act + string description = QueryStoreDiagnosticsWatchdog.DescribeReadonlyReason(65536 | (1 << 20)); + + // Assert + Assert.Contains("size limit", description, StringComparison.Ordinal); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs index 72b51591f6..14aff5b74b 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs @@ -3,11 +3,23 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using EnsureThat; + namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { + /// + /// Outcome of Showplan sanitization. + /// + /// + /// The constructor is private and instances are produced only through the factory methods below, because the + /// payload can embed literal parameter values taken from patient data. "Not verified but populated" is therefore + /// a PHI-leak shape and must be impossible to express: every failure factory forces to null and + /// the success factory refuses a null document. is derived rather than supplied so it can + /// never disagree with the payload it describes. + /// internal sealed class QueryPlanSanitizationResult { - internal QueryPlanSanitizationResult(string status, string xml, bool truncated, int originalLength, int sanitizedLength) + private QueryPlanSanitizationResult(string status, string xml, bool truncated, int originalLength, int sanitizedLength) { Status = status; Xml = xml; @@ -16,10 +28,19 @@ internal QueryPlanSanitizationResult(string status, string xml, bool truncated, SanitizedLength = sanitizedLength; } + /// + /// Gets the sanitization outcome, one of the status constants on . + /// internal string Status { get; } + /// + /// Gets the sanitized and verified Showplan XML, or null when sanitization did not succeed. + /// internal string Xml { get; } + /// + /// Gets a value indicating whether was truncated to the field cap. + /// internal bool Truncated { get; } /// @@ -32,5 +53,61 @@ internal QueryPlanSanitizationResult(string status, string xml, bool truncated, /// This is the value to compare against the field cap when is set. /// internal int SanitizedLength { get; } + + /// + /// Creates a successful result. The XML is required to be non-null and already verified free of parameter data. + /// + /// The sanitized, verified and field-capped Showplan XML. + /// The length of the raw Showplan XML as read from Query Store. + /// The length of the sanitized Showplan XML before truncation. + /// A result carrying the sanitized document. + internal static QueryPlanSanitizationResult Sanitized(string xml, int originalLength, int sanitizedLength) + { + EnsureArg.IsNotNull(xml, nameof(xml)); + EnsureArg.IsGte(originalLength, 0, nameof(originalLength)); + EnsureArg.IsGte(sanitizedLength, xml.Length, nameof(sanitizedLength)); + + return new QueryPlanSanitizationResult( + QueryPlanSanitizer.SanitizedStatus, + xml, + sanitizedLength > xml.Length, + originalLength, + sanitizedLength); + } + + /// + /// Creates a result for a plan that Query Store did not supply any Showplan XML for. + /// + /// A result with null XML. + internal static QueryPlanSanitizationResult PlanXmlUnavailable() + { + return new QueryPlanSanitizationResult(QueryPlanSanitizer.PlanXmlUnavailableStatus, null, false, 0, 0); + } + + /// + /// Creates a result for Showplan XML that could not be parsed. + /// + /// The length of the raw Showplan XML as read from Query Store. + /// A result with null XML. + internal static QueryPlanSanitizationResult InvalidXml(int originalLength) + { + EnsureArg.IsGte(originalLength, 0, nameof(originalLength)); + + return new QueryPlanSanitizationResult(QueryPlanSanitizer.InvalidXmlStatus, null, false, originalLength, 0); + } + + /// + /// Creates a result for Showplan XML in which parameter data survived removal. The document is discarded. + /// + /// The length of the raw Showplan XML as read from Query Store. + /// The length of the discarded document, retained so the loss is quantifiable. + /// A result with null XML. + internal static QueryPlanSanitizationResult VerificationFailed(int originalLength, int sanitizedLength) + { + EnsureArg.IsGte(originalLength, 0, nameof(originalLength)); + EnsureArg.IsGte(sanitizedLength, 0, nameof(sanitizedLength)); + + return new QueryPlanSanitizationResult(QueryPlanSanitizer.VerificationFailedStatus, null, false, originalLength, sanitizedLength); + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs index 555d8ea449..9e953efcb4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs @@ -11,6 +11,10 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { + /// + /// Removes Showplan parameter metadata, which can carry literal values taken from patient data, and verifies + /// that none of it survived before the plan is allowed out of the process. + /// internal static class QueryPlanSanitizer { internal const string SanitizedStatus = "Sanitized"; @@ -18,11 +22,17 @@ internal static class QueryPlanSanitizer internal const string InvalidXmlStatus = "InvalidXml"; internal const string VerificationFailedStatus = "VerificationFailed"; + /// + /// Removes parameter metadata from a Showplan document, verifies the removal, and caps the result. + /// + /// The raw Showplan XML as read from Query Store. + /// The maximum length of the returned XML. + /// The sanitization outcome. The XML is null unless removal was verified to have succeeded. internal static QueryPlanSanitizationResult Sanitize(string queryPlanXml, int maxLength) { if (string.IsNullOrEmpty(queryPlanXml)) { - return new QueryPlanSanitizationResult(PlanXmlUnavailableStatus, null, false, 0, 0); + return QueryPlanSanitizationResult.PlanXmlUnavailable(); } try @@ -55,31 +65,44 @@ internal static QueryPlanSanitizationResult Sanitize(string queryPlanXml, int ma var sanitizedXml = document.ToString(SaveOptions.DisableFormatting); var sanitizedLength = sanitizedXml.Length; - if (ContainsSensitiveParameterData(sanitizedXml)) + + // Verification re-inspects the tree after removal and fails closed. It is deliberately structural: + // Showplan embeds the original SQL in StatementText, so a text scan would drop any plan whose own + // query text happens to contain the literal string "ParameterList". + if (ContainsSensitiveParameterData(document)) { - return new QueryPlanSanitizationResult(VerificationFailedStatus, null, false, queryPlanXml.Length, sanitizedLength); + return QueryPlanSanitizationResult.VerificationFailed(queryPlanXml.Length, sanitizedLength); } maxLength = Math.Max(0, maxLength); - var truncated = sanitizedXml.Length > maxLength; - if (truncated) + if (sanitizedXml.Length > maxLength) { sanitizedXml = sanitizedXml.Substring(0, maxLength); } - return new QueryPlanSanitizationResult(SanitizedStatus, sanitizedXml, truncated, queryPlanXml.Length, sanitizedLength); + return QueryPlanSanitizationResult.Sanitized(sanitizedXml, queryPlanXml.Length, sanitizedLength); } catch (XmlException) { - return new QueryPlanSanitizationResult(InvalidXmlStatus, null, false, queryPlanXml.Length, 0); + return QueryPlanSanitizationResult.InvalidXml(queryPlanXml.Length); } } - private static bool ContainsSensitiveParameterData(string xml) + private static bool ContainsSensitiveParameterData(XDocument document) + { + // Matching is by local name and namespace-agnostic, exactly as the removal above, so a Showplan namespace + // change between SQL versions cannot let parameter data pass verification. + return document.Root != null && + document.Root.DescendantsAndSelf().Any(element => + IsSensitiveName(element.Name.LocalName) || + element.Attributes().Any(attribute => IsSensitiveName(attribute.Name.LocalName))); + } + + private static bool IsSensitiveName(string localName) { - return xml.Contains("ParameterList", StringComparison.OrdinalIgnoreCase) || - xml.Contains("ParameterCompiledValue", StringComparison.OrdinalIgnoreCase) || - xml.Contains("ParameterRuntimeValue", StringComparison.OrdinalIgnoreCase); + return string.Equals(localName, "ParameterList", StringComparison.OrdinalIgnoreCase) || + string.Equals(localName, "ParameterCompiledValue", StringComparison.OrdinalIgnoreCase) || + string.Equals(localName, "ParameterRuntimeValue", StringComparison.OrdinalIgnoreCase); } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs index 5d647e20d4..1895e4606f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs @@ -24,13 +24,27 @@ internal sealed class QueryStoreDiagnosticsWatchdog : WatchdogWait statistics were read for the plan. + internal const string WaitStatisticsAvailableStatus = "Available"; + + /// The wait query succeeded but returned no row for the plan. + internal const string WaitStatisticsUnavailableStatus = "Unavailable"; + + /// The wait query itself failed, so wait fields are missing because collection is broken. + internal const string WaitStatisticsFailedStatus = "Failed"; + private const string QueryStoreStateSql = @" +-- readonly_reason is a bitmask, and it is int here while neighbouring Query Store columns (query_id, plan_id, +-- count_executions, rows, rows_sampled, modification_counter) are bigint. That inconsistency is the trap: reading +-- this column with GetInt64 throws InvalidCastException at runtime, which no compiler or unit test will catch. SELECT actual_state_desc, readonly_reason FROM sys.database_query_store_options;"; private const string SlowQueriesSql = @" --- Query Store duration and CPU values are recorded in microseconds. --- Aggregate interval averages with their execution counts before converting to milliseconds. +-- Query Store duration and CPU values are recorded in MICROSECONDS while the emitted contract is in MILLISECONDS, +-- which is what every /1000.0 below is for. Query Store also stores PER-INTERVAL averages, so combining intervals +-- requires weighting each interval average by its count_executions first; an unweighted mean across intervals with +-- unequal execution counts is mathematically wrong. ;WITH PlanRuntimeStatistics AS ( SELECT @@ -63,6 +77,8 @@ SELECT TOP (@Top) queryText.query_sql_text, runtimeRollup.interval_start, runtimeRollup.interval_end +-- 'statistics' is a RESERVED T-SQL keyword and cannot be used as a table alias, which is why the rollup CTE is +-- aliased runtimeRollup rather than statistics. FROM PlanRuntimeStatistics AS runtimeRollup INNER JOIN sys.query_store_plan AS queryStorePlan ON runtimeRollup.plan_id = queryStorePlan.plan_id @@ -72,7 +88,9 @@ INNER JOIN sys.query_store_query_text AS queryText ON queryStoreQuery.query_text_id = queryText.query_text_id WHERE runtimeRollup.execution_count > 0 AND runtimeRollup.total_duration_microseconds / runtimeRollup.execution_count / 1000.0 >= @MinDurationMilliseconds - -- SQL Server does not preserve diagnostic marker comments in query_sql_text, so exclude every watchdog statement by catalog name. + -- Query Store does NOT preserve comments in query_sql_text, so a marker comment cannot be used to identify a + -- statement. The watchdog therefore excludes its own statements by catalog name, and the integration tests + -- identify their probe query by a GUID-derived result-column alias, which Query Store does preserve. -- The explicit case-insensitive collation also keeps the comparison correct on case-sensitive databases. AND queryText.query_sql_text COLLATE Latin1_General_CI_AS NOT LIKE N'%query_store%' AND queryText.query_sql_text COLLATE Latin1_General_CI_AS NOT LIKE N'%dm_db_stats_properties%' @@ -144,6 +162,8 @@ ELSE CONVERT(float, statisticsProperties.modification_counter) * 100.0 / statist statisticsObject.user_created, CONVERT(bit, CASE WHEN queryIndex.index_id IS NULL THEN 0 ELSE 1 END) AS is_from_index, statisticsObject.has_filter +-- 'statistics' is a RESERVED T-SQL keyword and cannot be used as a table alias, which is why sys.stats is +-- aliased statisticsObject rather than statistics. FROM sys.stats AS statisticsObject INNER JOIN sys.objects AS queryObject ON statisticsObject.object_id = queryObject.object_id @@ -152,7 +172,6 @@ INNER JOIN sys.tables AS queryTable LEFT JOIN sys.indexes AS queryIndex ON statisticsObject.object_id = queryIndex.object_id AND statisticsObject.stats_id = queryIndex.index_id --- OUTER APPLY retains statistics when dm_db_stats_properties has no row. OUTER APPLY sys.dm_db_stats_properties(statisticsObject.object_id, statisticsObject.stats_id) AS statisticsProperties WHERE queryObject.is_ms_shipped = 0 AND queryObject.type = 'U' @@ -233,23 +252,39 @@ protected override async Task RunWorkAsync(CancellationToken cancellationToken) } var lookbackPeriodSec = Math.Clamp(await GetNumberParameterByIdAsync(PeriodSecId, cancellationToken), 60d, 86400d); - var startTime = DateTimeOffset.UtcNow.AddSeconds(-lookbackPeriodSec); + var collectionTime = DateTimeOffset.UtcNow; + var startTime = collectionTime.AddSeconds(-lookbackPeriodSec); var queryStoreState = await GetQueryStoreStateAsync(cancellationToken); - if (queryStoreState == null || !string.Equals(queryStoreState.ActualState, "READ_WRITE", StringComparison.OrdinalIgnoreCase)) + if (queryStoreState == null) + { + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: sys.database_query_store_options returned no row, so Query Store is not configured on this database. Skipping collection."); + return; + } + + if (!string.Equals(queryStoreState.ActualState, "READ_WRITE", StringComparison.OrdinalIgnoreCase)) { _logger.LogWarning( - "QueryStoreDiagnosticsWatchdog: Query Store is unavailable for diagnostics. State={QueryStoreState}, ReadonlyReason={ReadonlyReason}", - queryStoreState?.ActualState ?? "unavailable", - queryStoreState?.ReadonlyReason); + "QueryStoreDiagnosticsWatchdog: Query Store is unavailable for diagnostics. State={QueryStoreState}, ReadonlyReason={ReadonlyReason}, ReadonlyReasonDescription={ReadonlyReasonDescription}", + queryStoreState.ActualState, + queryStoreState.ReadonlyReason, + DescribeReadonlyReason(queryStoreState.ReadonlyReason)); return; } + if (_configuration.SlowQueryCount <= 0) + { + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: SlowQueryCount is {SlowQueryCount}, which disables slow-query collection. Configure a positive value to collect slow queries.", + _configuration.SlowQueryCount); + } + var slowQueries = await GetSlowQueriesAsync(startTime, cancellationToken); var waitStatistics = await GetWaitStatisticsAsync(startTime, slowQueries, cancellationToken); foreach (var slowQuery in slowQueries) { - waitStatistics.TryGetValue(slowQuery.PlanId, out var wait); + waitStatistics.Waits.TryGetValue(slowQuery.PlanId, out var wait); var queryText = Truncate(slowQuery.QueryText); await _mediator.PublishAsync( new SlowQueryNotification @@ -267,6 +302,7 @@ await _mediator.PublishAsync( TotalWaitMilliseconds = wait?.TotalWaitMilliseconds, AverageWaitMilliseconds = wait == null ? null : wait.TotalWaitMilliseconds / slowQuery.ExecutionCount, TopWaitCategory = wait?.TopWaitCategory, + WaitStatisticsStatus = GetWaitStatisticsStatus(waitStatistics.Failed, wait), QueryText = queryText.Value, QueryTextTruncated = queryText.Truncated, QueryTextLength = queryText.OriginalLength, @@ -276,19 +312,50 @@ await _mediator.PublishAsync( cancellationToken); } - if (_configuration.IncludeQueryPlans && slowQueries.Count > 0) + var queryPlanCount = 0; + if (!_configuration.IncludeQueryPlans) { - await PublishQueryPlansAsync(slowQueries, cancellationToken); + _logger.LogInformation("QueryStoreDiagnosticsWatchdog: query plan collection is turned off by configuration (IncludeQueryPlans)."); + } + else if (slowQueries.Count > 0) + { + queryPlanCount = await PublishQueryPlansAsync(slowQueries, cancellationToken); } - if (_configuration.IncludeStatisticsHealth && _configuration.StatisticsHealthCount > 0) + var statisticsHealthCount = 0; + if (!_configuration.IncludeStatisticsHealth) + { + _logger.LogInformation("QueryStoreDiagnosticsWatchdog: statistics health collection is turned off by configuration (IncludeStatisticsHealth)."); + } + else if (_configuration.StatisticsHealthCount <= 0) { - await PublishStatisticsHealthAsync(cancellationToken); + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: StatisticsHealthCount is {StatisticsHealthCount}, which disables statistics health collection. Configure a positive value to collect statistics health.", + _configuration.StatisticsHealthCount); + } + else + { + statisticsHealthCount = await PublishStatisticsHealthAsync(cancellationToken); } + + // A completed tick logs unconditionally, including zero counts: without this, "the watchdog has been + // dead for three days" and "there were no slow queries" are indistinguishable downstream. + _logger.LogInformation( + "QueryStoreDiagnosticsWatchdog completed a collection. WindowStart={WindowStart}, WindowEnd={WindowEnd}, SlowQueries={SlowQueryCount}, QueryPlans={QueryPlanCount}, StatisticsHealth={StatisticsHealthCount}, WaitStatisticsFailed={WaitStatisticsFailed}", + startTime, + collectionTime, + slowQueries.Count, + queryPlanCount, + statisticsHealthCount, + waitStatistics.Failed); } catch (SqlException ex) when (ex.Number == 208) { - _logger.LogDebug(ex, "QueryStoreDiagnosticsWatchdog: Query Store diagnostics views are unavailable."); + // This filter covers the whole method body rather than each read, which loses the ability to name the + // failing statement. That is the accepted trade-off: the watchdog only runs when an operator enabled + // it in both configuration and dbo.Parameters, so "the views you asked me to read do not exist" is + // always operator-actionable and permanent, and per-read catches would be more churn than value. + _logger.LogWarning(ex, "QueryStoreDiagnosticsWatchdog: Query Store diagnostics views are unavailable, so no diagnostics can be collected."); } catch (SqlException ex) when (ex.Number == 229 || ex.Number == 262) { @@ -299,6 +366,12 @@ await _mediator.PublishAsync( private async Task GetQueryStoreStateAsync(CancellationToken cancellationToken) { await using var command = new SqlCommand(QueryStoreStateSql); + + // Every diagnostics read binds to the primary: isReadOnly would route to a read-only secondary when + // SupportsSqlReplicas is on, and Query Store state is primary-scoped, so a secondary reports READ_ONLY + // and the state gate below would silently disable collection forever. Replica routing is also decided + // per call, so the state check and the data reads could otherwise land on different servers and produce + // torn results. The cost is negligible: one collection per period, hourly by default. var states = await _sqlRetryService.ExecuteReaderAsync( command, reader => new QueryStoreState( @@ -306,8 +379,7 @@ private async Task GetQueryStoreStateAsync(CancellationToken ca reader.IsDBNull(1) ? (int?)null : reader.GetInt32(1)), _logger, "Failed to read Query Store state", - cancellationToken, - isReadOnly: true); + cancellationToken); return states.Count == 0 ? null : states[0]; } @@ -339,18 +411,17 @@ private async Task> GetSlowQueriesAsync(DateTimeO }, _logger, "Failed to read Query Store slow queries", - cancellationToken, - isReadOnly: true); + cancellationToken); } - private async Task> GetWaitStatisticsAsync( + private async Task<(Dictionary Waits, bool Failed)> GetWaitStatisticsAsync( DateTimeOffset startTime, IReadOnlyList slowQueries, CancellationToken cancellationToken) { if (slowQueries.Count == 0) { - return new Dictionary(); + return (new Dictionary(), false); } try @@ -367,19 +438,22 @@ private async Task> GetWaitStatisticsAsync( reader.IsDBNull(2) ? null : reader.GetString(2)), _logger, "Failed to read Query Store wait statistics", - cancellationToken, - isReadOnly: true); + cancellationToken); - return waits.ToDictionary(wait => wait.PlanId); + return (waits.ToDictionary(wait => wait.PlanId), false); } catch (SqlException ex) { - _logger.LogDebug(ex, "QueryStoreDiagnosticsWatchdog: Query Store wait statistics are unavailable for this collection."); - return new Dictionary(); + // SqlException is caught broadly on purpose: a transient wait-query failure must never abort the tick + // and suppress the runtime metrics, which are the primary signal. The cost of that breadth is that + // timeouts, deadlocks, permission denials and missing views all look alike here, so the failure is + // logged as a warning and surfaced on every notification as WaitStatisticsStatus = Failed. + _logger.LogWarning(ex, "QueryStoreDiagnosticsWatchdog: Query Store wait statistics could not be read for this collection. Wait fields will be empty."); + return (new Dictionary(), true); } } - private async Task PublishQueryPlansAsync(IReadOnlyList slowQueries, CancellationToken cancellationToken) + private async Task PublishQueryPlansAsync(IReadOnlyList slowQueries, CancellationToken cancellationToken) { await using var command = new SqlCommand(QueryPlansSql); command.Parameters.Add("@PlanIds", SqlDbType.NVarChar, -1).Value = string.Join(',', slowQueries.Select(query => query.PlanId)); @@ -391,14 +465,23 @@ private async Task PublishQueryPlansAsync(IReadOnlyList slowQue reader.IsDBNull(1) ? null : reader.GetString(1)), _logger, "Failed to read Query Store plans", - cancellationToken, - isReadOnly: true); + cancellationToken); var plansById = plans.ToDictionary(plan => plan.PlanId); foreach (var slowQuery in slowQueries) { plansById.TryGetValue(slowQuery.PlanId, out var queryPlan); var sanitizedPlan = QueryPlanSanitizer.Sanitize(queryPlan?.QueryPlan, MaxFieldLength); + if (!string.Equals(sanitizedPlan.Status, QueryPlanSanitizer.SanitizedStatus, StringComparison.Ordinal)) + { + // Without this, systematic sanitizer breakage looks exactly like "plans are simply unavailable" + // unless a downstream handler happens to surface SanitizationStatus. + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: query plan was not emitted because sanitization did not succeed. PlanId={PlanId}, SanitizationStatus={SanitizationStatus}", + slowQuery.PlanId, + sanitizedPlan.Status); + } + await _mediator.PublishAsync( new QueryPlanNotification { @@ -412,9 +495,11 @@ await _mediator.PublishAsync( }, cancellationToken); } + + return slowQueries.Count; } - private async Task PublishStatisticsHealthAsync(CancellationToken cancellationToken) + private async Task PublishStatisticsHealthAsync(CancellationToken cancellationToken) { await using var command = new SqlCommand(StatisticsHealthSql); command.Parameters.Add("@Top", SqlDbType.Int).Value = _configuration.StatisticsHealthCount; @@ -438,8 +523,7 @@ private async Task PublishStatisticsHealthAsync(CancellationToken cancellationTo }, _logger, "Failed to read statistics health", - cancellationToken, - isReadOnly: true); + cancellationToken); foreach (var statistic in statisticsHealth) { @@ -461,6 +545,8 @@ await _mediator.PublishAsync( }, cancellationToken); } + + return statisticsHealth.Count; } private async Task IsEnabledAsync(CancellationToken cancellationToken) @@ -469,6 +555,68 @@ private async Task IsEnabledAsync(CancellationToken cancellationToken) return value == 1; } + /// + /// Decodes the sys.database_query_store_options.readonly_reason bitmask into a readable reason list, + /// so an operator sees the cause rather than an integer to look up. Exposed as internal only for unit testing. + /// + /// The bitmask value, or null when no value was reported. + /// A comma-separated description of the set bits. + internal static string DescribeReadonlyReason(int? readonlyReason) + { + if (readonlyReason == null) + { + return "not reported"; + } + + if (readonlyReason.Value == 0) + { + return "none"; + } + + var reasons = new List(); + if ((readonlyReason.Value & 1) != 0) + { + reasons.Add("database is in read-only mode"); + } + + if ((readonlyReason.Value & 2) != 0) + { + reasons.Add("database is in single-user mode"); + } + + if ((readonlyReason.Value & 4) != 0) + { + reasons.Add("database is in emergency mode"); + } + + if ((readonlyReason.Value & 8) != 0) + { + reasons.Add("database is a secondary replica"); + } + + if ((readonlyReason.Value & 65536) != 0) + { + reasons.Add("Query Store has reached its size limit (MAX_STORAGE_SIZE_MB)"); + } + + if ((readonlyReason.Value & 131072) != 0) + { + reasons.Add("Query Store has reached the limit on the number of statements"); + } + + return reasons.Count == 0 ? "unrecognized reason" : string.Join(", ", reasons); + } + + private static string GetWaitStatisticsStatus(bool waitStatisticsFailed, WaitStatistics wait) + { + if (waitStatisticsFailed) + { + return WaitStatisticsFailedStatus; + } + + return wait == null ? WaitStatisticsUnavailableStatus : WaitStatisticsAvailableStatus; + } + private static TruncatedField Truncate(string value) { value ??= string.Empty; @@ -534,7 +682,10 @@ private sealed class StatisticsHealthResult private sealed class QueryStoreState { - // sys.database_query_store_options.readonly_reason is int, not bigint. + // Trap: sys.database_query_store_options.readonly_reason is int, NOT bigint, even though the neighbouring + // Query Store columns this watchdog reads (query_id, plan_id, SUM(count_executions), rows, rows_sampled, + // modification_counter) genuinely are bigint. Reading it with GetInt64 compiles and only fails at runtime + // with InvalidCastException, so it must be read with GetInt32 and held as int. internal QueryStoreState(string actualState, int? readonlyReason) { ActualState = actualState; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs index 5826d17d36..912adc7960 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Data; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Medino; @@ -30,6 +31,11 @@ public class QueryStoreDiagnosticsWatchdogTests : IClassFixture 0); Assert.True(slowQuery.QueryTextLength > 0); + // The probe runs a fixed number of times under a GUID alias, so the rollup across Query Store + // intervals must sum to exactly that count. + Assert.Equal(QueryExecutionCount, slowQuery.ExecutionCount); + + // Query Store records microseconds and the contract is milliseconds. The probe runs at MAXDOP 1 and + // is timed on the client, so the reported totals must sit inside the wall clock plus a tolerance. A + // missing /1000.0 would inflate these by three orders of magnitude and break the upper bound; a + // doubly applied one would sink them below MinDurationMilliseconds and the query would never appear. + double durationUpperBoundMilliseconds = probeElapsedMilliseconds + ProbeTimingToleranceMilliseconds; + Assert.InRange(slowQuery.TotalDurationMilliseconds, 1, durationUpperBoundMilliseconds); + Assert.InRange(slowQuery.AverageDurationMilliseconds, 1, durationUpperBoundMilliseconds); + Assert.InRange(slowQuery.MaxDurationMilliseconds, 1, durationUpperBoundMilliseconds); + Assert.InRange(slowQuery.TotalCpuMilliseconds, 0, durationUpperBoundMilliseconds); + Assert.InRange(slowQuery.AverageCpuMilliseconds, 0, durationUpperBoundMilliseconds); + Assert.Equal(slowQuery.TotalDurationMilliseconds / QueryExecutionCount, slowQuery.AverageDurationMilliseconds, 3); + Assert.True(slowQuery.TotalLogicalReads > 0); + + // Wait collection is best-effort and its failure is swallowed so that runtime metrics still publish. + // A status other than Failed is therefore the only proof that the wait SQL actually executed. + Assert.Contains( + slowQuery.WaitStatisticsStatus, + new[] { QueryStoreDiagnosticsWatchdog.WaitStatisticsAvailableStatus, QueryStoreDiagnosticsWatchdog.WaitStatisticsUnavailableStatus }); + if (string.Equals(slowQuery.WaitStatisticsStatus, QueryStoreDiagnosticsWatchdog.WaitStatisticsAvailableStatus, StringComparison.Ordinal)) + { + Assert.NotNull(slowQuery.TotalWaitMilliseconds); + Assert.NotNull(slowQuery.AverageWaitMilliseconds); + Assert.False(string.IsNullOrEmpty(slowQuery.TopWaitCategory)); + } + else + { + Assert.Null(slowQuery.TotalWaitMilliseconds); + } + QueryPlanNotification queryPlan = Assert.Single( notifications.FindAll(notification => notification is QueryPlanNotification) .ConvertAll(notification => (QueryPlanNotification)notification) @@ -171,7 +213,7 @@ private static async Task EnableAndVerifyQueryStoreAsync(SqlConnection connectio await ExecuteNonQueryAsync( connection, - "ALTER DATABASE CURRENT SET QUERY_STORE (OPERATION_MODE = READ_WRITE, QUERY_CAPTURE_MODE = ALL);", + "ALTER DATABASE CURRENT SET QUERY_STORE (OPERATION_MODE = READ_WRITE, QUERY_CAPTURE_MODE = ALL, WAIT_STATS_CAPTURE_MODE = ON);", cancellationToken); Assert.Equal("READ_WRITE", await GetQueryStoreStateAsync(connection, cancellationToken), ignoreCase: true); @@ -226,15 +268,21 @@ private static async Task DropProbeTableAsync(SqlConnection connection, string t await ExecuteNonQueryAsync(connection, $"DROP TABLE IF EXISTS dbo.[{tableName}];", cancellationToken); } - private static async Task ExecuteProbeQueryAsync(SqlConnection connection, string tableName, string queryAlias, CancellationToken cancellationToken) + private static async Task ExecuteProbeQueryAsync(SqlConnection connection, string tableName, string queryAlias, CancellationToken cancellationToken) { + // MAXDOP 1 keeps CPU time comparable with elapsed time, so the millisecond assertions have a meaningful + // upper bound rather than one inflated by an unknown degree of parallelism. + var stopwatch = Stopwatch.StartNew(); for (int execution = 0; execution < QueryExecutionCount; execution++) { await using SqlCommand command = connection.CreateCommand(); - command.CommandText = $"SELECT SUM(CONVERT(bigint, firstProbe.Id) * secondProbe.Id * thirdProbe.Id) AS [{queryAlias}] FROM dbo.[{tableName}] AS firstProbe CROSS JOIN dbo.[{tableName}] AS secondProbe CROSS JOIN dbo.[{tableName}] AS thirdProbe;"; + command.CommandText = $"SELECT SUM(CONVERT(bigint, firstProbe.Id) * secondProbe.Id * thirdProbe.Id) AS [{queryAlias}] FROM dbo.[{tableName}] AS firstProbe CROSS JOIN dbo.[{tableName}] AS secondProbe CROSS JOIN dbo.[{tableName}] AS thirdProbe OPTION (MAXDOP 1);"; object result = await command.ExecuteScalarAsync(cancellationToken); Assert.NotNull(result); } + + stopwatch.Stop(); + return stopwatch.Elapsed.TotalMilliseconds; } private static async Task WaitForQueryStoreCaptureAsync(SqlConnection connection, string queryAlias, CancellationToken cancellationToken) @@ -244,8 +292,11 @@ private static async Task WaitForQueryStoreCaptureAsync(SqlConnection connection await ExecuteNonQueryAsync(connection, "EXEC sys.sp_query_store_flush_db;", cancellationToken); await using SqlCommand command = connection.CreateCommand(); + + // Wait for every execution to be persisted, not merely the first, so the execution-count assertion + // cannot race the flush. command.CommandText = @" -SELECT COUNT_BIG(*) +SELECT ISNULL(SUM(runtimeStats.count_executions), 0) FROM sys.query_store_runtime_stats AS runtimeStats INNER JOIN sys.query_store_plan AS queryPlan ON runtimeStats.plan_id = queryPlan.plan_id @@ -257,8 +308,8 @@ WHERE queryText.query_sql_text LIKE @QueryTextPattern AND runtimeStats.execution_type = 0;"; command.Parameters.Add("@QueryTextPattern", SqlDbType.NVarChar, 256).Value = $"%{queryAlias}%"; - long capturedRuntimeStatisticsCount = Convert.ToInt64(await command.ExecuteScalarAsync(cancellationToken)); - if (capturedRuntimeStatisticsCount > 0) + long capturedExecutionCount = Convert.ToInt64(await command.ExecuteScalarAsync(cancellationToken)); + if (capturedExecutionCount >= QueryExecutionCount) { return; } @@ -266,7 +317,7 @@ WHERE queryText.query_sql_text LIKE @QueryTextPattern await Task.Delay(QueryStorePollInterval, cancellationToken); } - Assert.Fail("Query Store did not persist regular runtime statistics for the GUID-alias probe query after the supported flush and polling window."); + Assert.Fail("Query Store did not persist regular runtime statistics for every execution of the GUID-alias probe query after the supported flush and polling window."); } private static async Task ExecuteNonQueryAsync(SqlConnection connection, string commandText, CancellationToken cancellationToken) From 7c5a0a10c878b132a346cac0cfaa6265a9403876 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 21 Aug 2026 20:46:19 +0000 Subject: [PATCH 11/20] Correct diagnostics claims and cover the wait-failure path Round two of review findings, all verified against the source. Three statements were claiming more than the code delivers: - the design doc said a missing Query Store row is logged "naming the state and readonly_reason", which that branch cannot do because there is no row; the two branches are now described separately - it said wait capture being off yields Failed with a warning, when only a caught SqlException does; the ordinary empty result is Unavailable and silent - both the doc and the XML doc on QueryPlanSanitizationResult said "not verified but populated" is unconstructable. The success factory accepts any non-null string, so the guarantee that actually holds is narrower: no failure status can carry a payload, success refuses null, and Truncated is derived. The claim now matches that and the type is unchanged. Make two log lines carry the information they implied: - the missing-view handler spans the whole collection, so the aborting read can be the last one and slow queries may already have been published. It no longer says "no diagnostics can be collected". - the completion log's plan count returned the slow-query count unconditionally, so it always equalled its neighbour. It now counts plans that actually carried sanitized XML. DescribeReadonlyReason dropped unrecognized bits whenever a documented bit was set alongside them, so a state flag from a newer SQL Server would vanish. Unknown bits are now reported with the raw value. Remove a DTO that mirrored StatisticsHealthNotification property for property, and a truncation helper class serving one call site. A non-positive SlowQueryCount now skips the read rather than issuing TOP (0). Tests: the probe asserted Assert.Single over notifications grouped by plan_id while the capture poll summed across plans, so a recompile between the two probe executions would have produced two plans and an intermittent red build; it now sums. Statistics health asserted only NotEmpty over twelve positional reads, leaving column order unpinned; it now pins ordinals against deterministic full-scan statistics. The CPU lower bound moves off zero, and a new unit test drives the wait read to throw and asserts slow queries still publish with WaitStatisticsStatus of Failed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 14 +- ...ueryStoreDiagnosticsWaitStatisticsTests.cs | 125 +++++++++ .../QueryStoreReadonlyReasonTests.cs | 19 ++ .../AssemblyInfo.cs | 4 + .../Watchdogs/QueryPlanSanitizationResult.cs | 9 +- .../QueryStoreDiagnosticsWatchdog.cs | 243 +++++++++--------- .../QueryStoreDiagnosticsWatchdogTests.cs | 191 ++++++++++---- 7 files changed, 426 insertions(+), 179 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 473267072f..0a85043a57 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -113,7 +113,7 @@ Truncation is applied **after** sanitization and verification, never before, so ### Query Store state -The watchdog reads `sys.database_query_store_options` and proceeds only when `actual_state_desc` is `READ_WRITE`. Any other state, or no row at all, is logged as a warning naming the state and `readonly_reason`, and the tick is skipped. +The watchdog reads `sys.database_query_store_options` and proceeds only when `actual_state_desc` is `READ_WRITE`. A row reporting any other state is logged as a warning naming that state together with the decoded `readonly_reason`, and the tick is skipped. When the view returns no row at all there is no state and no reason to report, so that case is logged as its own warning saying that Query Store is not configured on the database, and the tick is skipped. **The watchdog never issues `ALTER DATABASE`.** Turning Query Store on is a database-scoped configuration change with its own permission and blast-radius considerations, and it is on by default in Azure SQL Database. Enabling it stays an explicit operator action. @@ -131,7 +131,9 @@ The watchdog's own Query Store reads are excluded by filtering out query text re ### Wait statistics -Wait statistics are collected by a **separate, best-effort** query and merged in C#. A failure — most commonly `sys.query_store_wait_stats` being unavailable, or wait capture being off — is logged as a warning, leaves the wait fields null, and sets `WaitStatisticsStatus` to `Failed`. Runtime results are still emitted. +Wait statistics are collected by a **separate, best-effort** query and merged in C#. A caught `SqlException` — the wait view being unavailable, a timeout, a deadlock, or a permission denial — is logged as a warning, leaves the wait fields null, and sets `WaitStatisticsStatus` to `Failed`. Runtime results are still emitted. + +`Failed` is reserved for that case: a wait read that actually broke. The ordinary outcomes of wait capture being turned off, or of no waits having accrued for a plan inside the window, are an empty result set rather than a failure — those plans carry `WaitStatisticsStatus` of `Unavailable`, with null wait fields and no warning logged. `SqlException` is caught broadly there on purpose, so that a transient wait-query failure cannot abort the tick and suppress the runtime metrics. The trade-off accepted is that timeouts, deadlocks, permission denials and missing views are not distinguished from one another at that point; the warning carries the exception, and the notification carries the status. @@ -147,6 +149,8 @@ Modification percentage is left null when the row count is null or zero rather t `WatchdogsBackgroundService` cancels **every** watchdog if any one of them fails. A diagnostic feature must never be able to take down transaction or cleanup watchdogs, so the collection body contains its own failures: missing views and permission denials are logged and the tick returns rather than propagating. +The missing-view handler spans the whole collection rather than each individual read, so the aborting read can be the last one, after slow queries and plans have already been published. Its warning is worded to hold in that case too: it reports that collection was aborted and that whatever had already been emitted was still published, rather than claiming that nothing was collected. + That containment covers **per-tick collection only**. `Watchdog.ExecuteAsync` awaits `InitParamsAsync` *before* and *outside* `FhirTimer`'s per-tick catch, so a throw during initialization — seeding the `dbo.Parameters` rows — still faults the watchdog task, and `WatchdogsBackgroundService` cancels the rest. This is a **pre-existing property of the shared watchdog framework**, not something this feature introduces: `DefragWatchdog` initializes with the identical insert pattern. It is documented here rather than worked around, because changing the shared framework is out of scope for a diagnostics feature. ### Reading the primary @@ -159,9 +163,9 @@ The cost is one collection per period against the primary — hourly by default ### Configuration that disables collection -A non-positive `SlowQueryCount` or `StatisticsHealthCount` reduces the corresponding query to `TOP (0)` or skips the section entirely, which is indistinguishable from a healthy empty result. Both cases are logged as a warning once per tick, and a section turned off deliberately through `IncludeQueryPlans` or `IncludeStatisticsHealth` is logged at information level. Misconfiguration is never fatal: a diagnostics feature must not fail the host. +A non-positive `SlowQueryCount` or `StatisticsHealthCount` disables the corresponding section: the round-trip is skipped entirely rather than issued as a `TOP (0)` query whose empty result would be indistinguishable from a healthy one. Both cases are logged as a warning once per tick, and a section turned off deliberately through `IncludeQueryPlans` or `IncludeStatisticsHealth` is logged at information level. Misconfiguration is never fatal: a diagnostics feature must not fail the host. -A completed tick logs one information-level line carrying the collection window and the counts of slow queries, plans and statistics rows published, **including zeros**, so that "the watchdog has been dead for three days" is distinguishable from "there were no slow queries". +A completed tick logs one information-level line carrying the collection window and the counts of slow queries, plans and statistics rows published, **including zeros**, so that "the watchdog has been dead for three days" is distinguishable from "there were no slow queries". The plan count is the number of plans that actually carried sanitized XML, not the number of plan notifications published, so it is deliberately lower than the slow-query count whenever Query Store held no plan for a query or sanitization rejected one. ## Sanitization @@ -177,7 +181,7 @@ The sanitizer: Step 4 is defence in depth: the plan is never emitted on the strength of the removal logic alone. It is structural because Showplan embeds the original SQL in `StatementText`, so a text scan would drop — silently and permanently — any plan whose own query text happens to contain the literal string `ParameterList`. -`QueryPlanSanitizationResult` is constructed only through static factories, so "not verified but populated" is unconstructable rather than merely unused: every failure factory forces the XML to null, the success factory refuses a null document, and the truncation flag is derived from the payload rather than supplied alongside it. +`QueryPlanSanitizationResult` is constructed only through static factories. The factories do not re-verify the document — the success factory trusts its caller for that — but they do constrain the result's shape: every failure factory forces the XML to null, so no failure status can be paired with a payload; the success factory refuses a null document; and the truncation flag is derived from the payload rather than supplied alongside it. ### Disclosure boundary diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs new file mode 100644 index 0000000000..93995b307d --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs @@ -0,0 +1,125 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Medino; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Metrics; +using Microsoft.Health.Fhir.SqlServer.Features.Storage; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Storage; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +{ + /// + /// Covers the branch that reports a broken wait collection. The integration test can only assert that the status + /// is not Failed, because it cannot make a live wait read fail, so without this the branch that exists + /// specifically to make that breakage visible would never be executed by the suite. + /// + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public class QueryStoreDiagnosticsWaitStatisticsTests + { + private const int DeadlockErrorNumber = 1205; + + [Fact] + public async Task GivenAFailingWaitStatisticsRead_WhenCollecting_ThenSlowQueriesArePublishedWithFailedWaitStatus() + { + // Arrange + var sqlRetryService = Substitute.For(); + var mediator = Substitute.For(); + var published = new List(); + mediator.When(x => x.PublishAsync(Arg.Any(), Arg.Any())) + .Do(info => published.Add((SlowQueryNotification)info[0])); + + var slowQuery = new QueryStoreDiagnosticsWatchdog.SlowQueryResult + { + QueryId = 11, + PlanId = 22, + ExecutionCount = 4, + TotalDurationMilliseconds = 400, + AverageDurationMilliseconds = 100, + MaxDurationMilliseconds = 150, + TotalCpuMilliseconds = 200, + AverageCpuMilliseconds = 50, + TotalLogicalReads = 80, + AverageLogicalReads = 20, + QueryText = "SELECT 1", + IntervalStart = DateTimeOffset.UtcNow.AddMinutes(-5), + IntervalEnd = DateTimeOffset.UtcNow, + }; + + sqlRetryService + .ExecuteReaderAsync( + Arg.Any(), + Arg.Any>(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new List { slowQuery }); + + sqlRetryService + .ExecuteReaderAsync( + Arg.Any(), + Arg.Any>(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns>( + _ => throw SqlExceptionFactory.GetSqlException(DeadlockErrorNumber, "Transaction was deadlocked on lock resources.")); + + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(sqlRetryService, mediator); + + // Act + await watchdog.CollectDiagnosticsAsync(DateTimeOffset.UtcNow.AddHours(-1), DateTimeOffset.UtcNow, CancellationToken.None); + + // Assert + // The runtime metrics are the primary signal, so a broken wait read must not suppress them... + SlowQueryNotification notification = Assert.Single(published); + Assert.Equal(slowQuery.QueryId, notification.QueryId); + Assert.Equal(slowQuery.PlanId, notification.PlanId); + Assert.Equal(slowQuery.TotalDurationMilliseconds, notification.TotalDurationMilliseconds); + + // ...and the breakage must be visible on the notification rather than looking like "this plan waited on + // nothing", which is what an Unavailable status would mean. + Assert.Equal(QueryStoreDiagnosticsWatchdog.WaitStatisticsFailedStatus, notification.WaitStatisticsStatus); + Assert.Null(notification.TotalWaitMilliseconds); + Assert.Null(notification.AverageWaitMilliseconds); + Assert.Null(notification.TopWaitCategory); + } + + private static QueryStoreDiagnosticsWatchdog CreateWatchdog(ISqlRetryService sqlRetryService, IMediator mediator) + { + var configuration = new WatchdogConfiguration(); + configuration.QueryStoreDiagnostics.Enabled = true; + configuration.QueryStoreDiagnostics.SlowQueryCount = 10; + configuration.QueryStoreDiagnostics.MinDurationMilliseconds = 1; + + // The plan and statistics sections are turned off so that the only reads this test has to stand up are + // the two it is about. + configuration.QueryStoreDiagnostics.IncludeQueryPlans = false; + configuration.QueryStoreDiagnostics.IncludeStatisticsHealth = false; + + return new QueryStoreDiagnosticsWatchdog( + sqlRetryService, + NullLogger.Instance, + mediator, + Options.Create(configuration)); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs index bf53037995..d07e257c9d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Globalization; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; @@ -77,5 +78,23 @@ public void GivenTheSizeLimitBitCombinedWithAnUndocumentedBit_WhenDescribed_Then // Assert Assert.Contains("size limit", description, StringComparison.Ordinal); } + + [Fact] + public void GivenADocumentedBitCombinedWithAnUndocumentedBit_WhenDescribed_ThenAlsoReportsTheUndocumentedBitAndTheRawValue() + { + // Arrange + const int undocumentedBit = 1 << 20; + const int mask = 65536 | undocumentedBit; + + // Act + string description = QueryStoreDiagnosticsWatchdog.DescribeReadonlyReason(mask); + + // Assert + // A state flag this code does not know about must not be swallowed just because a documented bit happened + // to be set alongside it: the raw value is what an operator takes to the SQL Server documentation. + Assert.Contains("Query Store has reached its size limit (MAX_STORAGE_SIZE_MB)", description, StringComparison.Ordinal); + Assert.Contains(undocumentedBit.ToString(CultureInfo.InvariantCulture), description, StringComparison.Ordinal); + Assert.Contains(mask.ToString(CultureInfo.InvariantCulture), description, StringComparison.Ordinal); + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.SqlServer/AssemblyInfo.cs index 7122f10988..7d6da185af 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/AssemblyInfo.cs @@ -7,6 +7,10 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.SqlServer.UnitTests")] + +// Castle DynamicProxy backs NSubstitute; without this it cannot produce a substitute value for a generic type +// closed over an internal type, which is what mocking ISqlRetryService reads in the watchdogs requires. +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Stu3.Tests.Integration")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.R4.Tests.Integration")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.R4B.Tests.Integration")] diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs index 14aff5b74b..6364a7d553 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs @@ -12,10 +12,11 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs /// /// /// The constructor is private and instances are produced only through the factory methods below, because the - /// payload can embed literal parameter values taken from patient data. "Not verified but populated" is therefore - /// a PHI-leak shape and must be impossible to express: every failure factory forces to null and - /// the success factory refuses a null document. is derived rather than supplied so it can - /// never disagree with the payload it describes. + /// payload can embed literal parameter values taken from patient data. The factories do not themselves verify + /// the document — trusts its caller for that — but they do guarantee the shape of the + /// result: every failure factory forces to null, so no failure status can be paired with a + /// payload, the success factory refuses a null document, and is derived from the + /// payload rather than supplied alongside it, so it cannot contradict what it describes. /// internal sealed class QueryPlanSanitizationResult { diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs index 1895e4606f..bb92f30980 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs @@ -272,20 +272,60 @@ protected override async Task RunWorkAsync(CancellationToken cancellationToken) return; } - if (_configuration.SlowQueryCount <= 0) - { - _logger.LogWarning( - "QueryStoreDiagnosticsWatchdog: SlowQueryCount is {SlowQueryCount}, which disables slow-query collection. Configure a positive value to collect slow queries.", - _configuration.SlowQueryCount); - } + await CollectDiagnosticsAsync(startTime, collectionTime, cancellationToken); + } + catch (SqlException ex) when (ex.Number == 208) + { + // This filter covers the whole method body rather than each read, which loses the ability to name the + // failing statement. That is the accepted trade-off: the watchdog only runs when an operator enabled + // it in both configuration and dbo.Parameters, so "the views you asked me to read do not exist" is + // always operator-actionable and permanent, and per-read catches would be more churn than value. + // Because the filter spans every read, the missing view can be the last one, after slow queries and + // plans have already been published; the message is therefore deliberately worded to be true of a + // partial tick as well as of one that emitted nothing. + _logger.LogWarning(ex, "QueryStoreDiagnosticsWatchdog: collection was aborted because a required Query Store or statistics view is unavailable. Any diagnostics already emitted during this collection were still published."); + } + catch (SqlException ex) when (ex.Number == 229 || ex.Number == 262) + { + _logger.LogWarning(ex, "QueryStoreDiagnosticsWatchdog: SQL permissions do not allow diagnostics collection."); + } + } - var slowQueries = await GetSlowQueriesAsync(startTime, cancellationToken); + /// + /// Runs the collection itself, once the configuration, runtime and Query Store state gates have all passed. + /// Separated from so that the reads and the notifications they produce are + /// reachable from unit tests: the gates above read dbo.Parameters through + /// , + /// which materializes its value inside a callback executed against a live and so + /// cannot be substituted. Exception handling deliberately stays in the caller. + /// + /// The start of the collection window. + /// The end of the collection window. + /// The cancellation token. + /// A task representing the asynchronous operation. + internal async Task CollectDiagnosticsAsync(DateTimeOffset startTime, DateTimeOffset collectionTime, CancellationToken cancellationToken) + { + IReadOnlyList slowQueries = Array.Empty(); + var waitStatisticsFailed = false; + if (_configuration.SlowQueryCount <= 0) + { + // Skipping the round-trip rather than running it with TOP (0) keeps this degenerate configuration + // reading the same way as the StatisticsHealthCount one below. + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: SlowQueryCount is {SlowQueryCount}, which disables slow-query collection. Configure a positive value to collect slow queries.", + _configuration.SlowQueryCount); + } + else + { + slowQueries = await GetSlowQueriesAsync(startTime, cancellationToken); var waitStatistics = await GetWaitStatisticsAsync(startTime, slowQueries, cancellationToken); + waitStatisticsFailed = waitStatistics.Failed; foreach (var slowQuery in slowQueries) { waitStatistics.Waits.TryGetValue(slowQuery.PlanId, out var wait); - var queryText = Truncate(slowQuery.QueryText); + var queryText = slowQuery.QueryText; + var queryTextTruncated = queryText.Length > MaxFieldLength; await _mediator.PublishAsync( new SlowQueryNotification { @@ -303,64 +343,54 @@ await _mediator.PublishAsync( AverageWaitMilliseconds = wait == null ? null : wait.TotalWaitMilliseconds / slowQuery.ExecutionCount, TopWaitCategory = wait?.TopWaitCategory, WaitStatisticsStatus = GetWaitStatisticsStatus(waitStatistics.Failed, wait), - QueryText = queryText.Value, - QueryTextTruncated = queryText.Truncated, - QueryTextLength = queryText.OriginalLength, + QueryText = queryTextTruncated ? queryText.Substring(0, MaxFieldLength) : queryText, + QueryTextTruncated = queryTextTruncated, + QueryTextLength = queryText.Length, IntervalStart = slowQuery.IntervalStart, IntervalEnd = slowQuery.IntervalEnd, }, cancellationToken); } + } - var queryPlanCount = 0; - if (!_configuration.IncludeQueryPlans) - { - _logger.LogInformation("QueryStoreDiagnosticsWatchdog: query plan collection is turned off by configuration (IncludeQueryPlans)."); - } - else if (slowQueries.Count > 0) - { - queryPlanCount = await PublishQueryPlansAsync(slowQueries, cancellationToken); - } - - var statisticsHealthCount = 0; - if (!_configuration.IncludeStatisticsHealth) - { - _logger.LogInformation("QueryStoreDiagnosticsWatchdog: statistics health collection is turned off by configuration (IncludeStatisticsHealth)."); - } - else if (_configuration.StatisticsHealthCount <= 0) - { - _logger.LogWarning( - "QueryStoreDiagnosticsWatchdog: StatisticsHealthCount is {StatisticsHealthCount}, which disables statistics health collection. Configure a positive value to collect statistics health.", - _configuration.StatisticsHealthCount); - } - else - { - statisticsHealthCount = await PublishStatisticsHealthAsync(cancellationToken); - } + var queryPlanCount = 0; + if (!_configuration.IncludeQueryPlans) + { + _logger.LogInformation("QueryStoreDiagnosticsWatchdog: query plan collection is turned off by configuration (IncludeQueryPlans)."); + } + else if (slowQueries.Count > 0) + { + queryPlanCount = await PublishQueryPlansAsync(slowQueries, cancellationToken); + } - // A completed tick logs unconditionally, including zero counts: without this, "the watchdog has been - // dead for three days" and "there were no slow queries" are indistinguishable downstream. - _logger.LogInformation( - "QueryStoreDiagnosticsWatchdog completed a collection. WindowStart={WindowStart}, WindowEnd={WindowEnd}, SlowQueries={SlowQueryCount}, QueryPlans={QueryPlanCount}, StatisticsHealth={StatisticsHealthCount}, WaitStatisticsFailed={WaitStatisticsFailed}", - startTime, - collectionTime, - slowQueries.Count, - queryPlanCount, - statisticsHealthCount, - waitStatistics.Failed); + var statisticsHealthCount = 0; + if (!_configuration.IncludeStatisticsHealth) + { + _logger.LogInformation("QueryStoreDiagnosticsWatchdog: statistics health collection is turned off by configuration (IncludeStatisticsHealth)."); } - catch (SqlException ex) when (ex.Number == 208) + else if (_configuration.StatisticsHealthCount <= 0) { - // This filter covers the whole method body rather than each read, which loses the ability to name the - // failing statement. That is the accepted trade-off: the watchdog only runs when an operator enabled - // it in both configuration and dbo.Parameters, so "the views you asked me to read do not exist" is - // always operator-actionable and permanent, and per-read catches would be more churn than value. - _logger.LogWarning(ex, "QueryStoreDiagnosticsWatchdog: Query Store diagnostics views are unavailable, so no diagnostics can be collected."); + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: StatisticsHealthCount is {StatisticsHealthCount}, which disables statistics health collection. Configure a positive value to collect statistics health.", + _configuration.StatisticsHealthCount); } - catch (SqlException ex) when (ex.Number == 229 || ex.Number == 262) + else { - _logger.LogWarning(ex, "QueryStoreDiagnosticsWatchdog: SQL permissions do not allow diagnostics collection."); + statisticsHealthCount = await PublishStatisticsHealthAsync(cancellationToken); } + + // A completed tick logs unconditionally, including zero counts: without this, "the watchdog has been + // dead for three days" and "there were no slow queries" are indistinguishable downstream. QueryPlans + // counts the plans that actually carried sanitized XML, so it is deliberately lower than SlowQueries + // whenever Query Store had no plan for a query or sanitization rejected one. + _logger.LogInformation( + "QueryStoreDiagnosticsWatchdog completed a collection. WindowStart={WindowStart}, WindowEnd={WindowEnd}, SlowQueries={SlowQueryCount}, QueryPlans={QueryPlanCount}, StatisticsHealth={StatisticsHealthCount}, WaitStatisticsFailed={WaitStatisticsFailed}", + startTime, + collectionTime, + slowQueries.Count, + queryPlanCount, + statisticsHealthCount, + waitStatisticsFailed); } private async Task GetQueryStoreStateAsync(CancellationToken cancellationToken) @@ -467,6 +497,7 @@ private async Task PublishQueryPlansAsync(IReadOnlyList sl "Failed to read Query Store plans", cancellationToken); var plansById = plans.ToDictionary(plan => plan.PlanId); + var publishedPlanCount = 0; foreach (var slowQuery in slowQueries) { @@ -494,9 +525,17 @@ await _mediator.PublishAsync( SanitizationStatus = sanitizedPlan.Status, }, cancellationToken); + + // A notification is published for every slow query, including the ones with no usable plan, but only + // the ones that carried XML are counted: a count that always equalled the slow-query count would tell + // an operator nothing about whether plans are actually arriving. + if (sanitizedPlan.Xml != null) + { + publishedPlanCount++; + } } - return slowQueries.Count; + return publishedPlanCount; } private async Task PublishStatisticsHealthAsync(CancellationToken cancellationToken) @@ -504,9 +543,11 @@ private async Task PublishStatisticsHealthAsync(CancellationToken cancellat await using var command = new SqlCommand(StatisticsHealthSql); command.Parameters.Add("@Top", SqlDbType.Int).Value = _configuration.StatisticsHealthCount; + // The reader projects straight into the notification contract: an intermediate DTO here would be a + // property-for-property copy of it and nothing else. var statisticsHealth = await _sqlRetryService.ExecuteReaderAsync( command, - reader => new StatisticsHealthResult + reader => new StatisticsHealthNotification { SchemaName = reader.GetString(0), TableName = reader.GetString(1), @@ -527,23 +568,7 @@ private async Task PublishStatisticsHealthAsync(CancellationToken cancellat foreach (var statistic in statisticsHealth) { - await _mediator.PublishAsync( - new StatisticsHealthNotification - { - SchemaName = statistic.SchemaName, - TableName = statistic.TableName, - StatisticsName = statistic.StatisticsName, - LastUpdated = statistic.LastUpdated, - Rows = statistic.Rows, - RowsSampled = statistic.RowsSampled, - ModificationCounter = statistic.ModificationCounter, - ModificationPercent = statistic.ModificationPercent, - IsAutoCreated = statistic.IsAutoCreated, - IsUserCreated = statistic.IsUserCreated, - IsFromIndex = statistic.IsFromIndex, - HasFilter = statistic.HasFilter, - }, - cancellationToken); + await _mediator.PublishAsync(statistic, cancellationToken); } return statisticsHealth.Count; @@ -563,6 +588,11 @@ private async Task IsEnabledAsync(CancellationToken cancellationToken) /// A comma-separated description of the set bits. internal static string DescribeReadonlyReason(int? readonlyReason) { + // Every bit this method knows how to name. An unrecognized bit is reported rather than dropped, so a + // state flag introduced by a newer SQL Server reaches the operator instead of vanishing behind whichever + // documented bits happened to be set alongside it. + const int knownReasonMask = 1 | 2 | 4 | 8 | 65536 | 131072; + if (readonlyReason == null) { return "not reported"; @@ -604,7 +634,18 @@ internal static string DescribeReadonlyReason(int? readonlyReason) reasons.Add("Query Store has reached the limit on the number of statements"); } - return reasons.Count == 0 ? "unrecognized reason" : string.Join(", ", reasons); + if (reasons.Count == 0) + { + return "unrecognized reason"; + } + + var unrecognizedBits = readonlyReason.Value & ~knownReasonMask; + if (unrecognizedBits != 0) + { + reasons.Add(FormattableString.Invariant($"unrecognized reason bits {unrecognizedBits} (readonly_reason = {readonlyReason.Value})")); + } + + return string.Join(", ", reasons); } private static string GetWaitStatisticsStatus(bool waitStatisticsFailed, WaitStatistics wait) @@ -617,14 +658,7 @@ private static string GetWaitStatisticsStatus(bool waitStatisticsFailed, WaitSta return wait == null ? WaitStatisticsUnavailableStatus : WaitStatisticsAvailableStatus; } - private static TruncatedField Truncate(string value) - { - value ??= string.Empty; - var truncated = value.Length > MaxFieldLength; - return new TruncatedField(truncated ? value.Substring(0, MaxFieldLength) : value, truncated, value.Length); - } - - private sealed class SlowQueryResult + internal sealed class SlowQueryResult { internal long QueryId { get; set; } @@ -653,33 +687,6 @@ private sealed class SlowQueryResult internal DateTimeOffset IntervalEnd { get; set; } } - private sealed class StatisticsHealthResult - { - internal string SchemaName { get; set; } - - internal string TableName { get; set; } - - internal string StatisticsName { get; set; } - - internal DateTimeOffset? LastUpdated { get; set; } - - internal long? Rows { get; set; } - - internal long? RowsSampled { get; set; } - - internal long? ModificationCounter { get; set; } - - internal double? ModificationPercent { get; set; } - - internal bool IsAutoCreated { get; set; } - - internal bool IsUserCreated { get; set; } - - internal bool IsFromIndex { get; set; } - - internal bool HasFilter { get; set; } - } - private sealed class QueryStoreState { // Trap: sys.database_query_store_options.readonly_reason is int, NOT bigint, even though the neighbouring @@ -697,7 +704,7 @@ internal QueryStoreState(string actualState, int? readonlyReason) internal int? ReadonlyReason { get; } } - private sealed class WaitStatistics + internal sealed class WaitStatistics { internal WaitStatistics(long planId, double totalWaitMilliseconds, string topWaitCategory) { @@ -725,21 +732,5 @@ internal QueryPlanResult(long planId, string queryPlan) internal string QueryPlan { get; } } - - private sealed class TruncatedField - { - internal TruncatedField(string value, bool truncated, int originalLength) - { - Value = value; - Truncated = truncated; - OriginalLength = originalLength; - } - - internal string Value { get; } - - internal bool Truncated { get; } - - internal int OriginalLength { get; } - } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs index 912adc7960..b72a7cdea0 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Data; using System.Diagnostics; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Medino; @@ -32,6 +33,12 @@ public class QueryStoreDiagnosticsWatchdogTests : IClassFixture notification is SlowQueryNotification) - .ConvertAll(notification => (SlowQueryNotification)notification) - .FindAll(notification => notification.QueryText.Contains(queryAlias, StringComparison.Ordinal))); - Assert.True(slowQuery.QueryId > 0); - Assert.True(slowQuery.PlanId > 0); - Assert.True(slowQuery.QueryTextLength > 0); + // The probe query is grouped by plan_id, and a recompile between executions would produce a second + // plan and therefore a second notification. That is a legitimate outcome, so the assertions are on + // the whole matching set: what must hold is that the executions add up. + List probeNotifications = notifications + .OfType() + .Where(notification => notification.QueryText.Contains(queryAlias, StringComparison.Ordinal)) + .ToList(); + Assert.NotEmpty(probeNotifications); // The probe runs a fixed number of times under a GUID alias, so the rollup across Query Store - // intervals must sum to exactly that count. - Assert.Equal(QueryExecutionCount, slowQuery.ExecutionCount); + // intervals and plans must sum to exactly that count. + Assert.Equal((long)QueryExecutionCount, probeNotifications.Sum(notification => notification.ExecutionCount)); // Query Store records microseconds and the contract is milliseconds. The probe runs at MAXDOP 1 and // is timed on the client, so the reported totals must sit inside the wall clock plus a tolerance. A // missing /1000.0 would inflate these by three orders of magnitude and break the upper bound; a // doubly applied one would sink them below MinDurationMilliseconds and the query would never appear. double durationUpperBoundMilliseconds = probeElapsedMilliseconds + ProbeTimingToleranceMilliseconds; - Assert.InRange(slowQuery.TotalDurationMilliseconds, 1, durationUpperBoundMilliseconds); - Assert.InRange(slowQuery.AverageDurationMilliseconds, 1, durationUpperBoundMilliseconds); - Assert.InRange(slowQuery.MaxDurationMilliseconds, 1, durationUpperBoundMilliseconds); - Assert.InRange(slowQuery.TotalCpuMilliseconds, 0, durationUpperBoundMilliseconds); - Assert.InRange(slowQuery.AverageCpuMilliseconds, 0, durationUpperBoundMilliseconds); - Assert.Equal(slowQuery.TotalDurationMilliseconds / QueryExecutionCount, slowQuery.AverageDurationMilliseconds, 3); - Assert.True(slowQuery.TotalLogicalReads > 0); - - // Wait collection is best-effort and its failure is swallowed so that runtime metrics still publish. - // A status other than Failed is therefore the only proof that the wait SQL actually executed. - Assert.Contains( - slowQuery.WaitStatisticsStatus, - new[] { QueryStoreDiagnosticsWatchdog.WaitStatisticsAvailableStatus, QueryStoreDiagnosticsWatchdog.WaitStatisticsUnavailableStatus }); - if (string.Equals(slowQuery.WaitStatisticsStatus, QueryStoreDiagnosticsWatchdog.WaitStatisticsAvailableStatus, StringComparison.Ordinal)) - { - Assert.NotNull(slowQuery.TotalWaitMilliseconds); - Assert.NotNull(slowQuery.AverageWaitMilliseconds); - Assert.False(string.IsNullOrEmpty(slowQuery.TopWaitCategory)); - } - else + foreach (SlowQueryNotification slowQuery in probeNotifications) { - Assert.Null(slowQuery.TotalWaitMilliseconds); + Assert.True(slowQuery.QueryId > 0); + Assert.True(slowQuery.PlanId > 0); + Assert.True(slowQuery.QueryTextLength > 0); + Assert.True(slowQuery.ExecutionCount > 0); + + Assert.InRange(slowQuery.TotalDurationMilliseconds, 1, durationUpperBoundMilliseconds); + Assert.InRange(slowQuery.AverageDurationMilliseconds, 1, durationUpperBoundMilliseconds); + Assert.InRange(slowQuery.MaxDurationMilliseconds, 1, durationUpperBoundMilliseconds); + + // The probe is a three-way cross join at MAXDOP 1, so its CPU is provably non-trivial: a lower + // bound of zero would let a regression that zeroed CPU entirely pass. + Assert.InRange(slowQuery.TotalCpuMilliseconds, 1, durationUpperBoundMilliseconds); + Assert.InRange(slowQuery.AverageCpuMilliseconds, 1, durationUpperBoundMilliseconds); + + // Query Store stores per-interval averages, so the emitted average must be the count-weighted + // one rather than an unweighted mean across intervals. + Assert.Equal(slowQuery.TotalDurationMilliseconds / slowQuery.ExecutionCount, slowQuery.AverageDurationMilliseconds, 3); + Assert.Equal(slowQuery.TotalCpuMilliseconds / slowQuery.ExecutionCount, slowQuery.AverageCpuMilliseconds, 3); + Assert.True(slowQuery.TotalLogicalReads > 0); + + // Wait collection is best-effort and its failure is swallowed so that runtime metrics still + // publish. A status other than Failed is therefore the only proof that the wait SQL executed. + Assert.Contains( + slowQuery.WaitStatisticsStatus, + new[] { QueryStoreDiagnosticsWatchdog.WaitStatisticsAvailableStatus, QueryStoreDiagnosticsWatchdog.WaitStatisticsUnavailableStatus }); + if (string.Equals(slowQuery.WaitStatisticsStatus, QueryStoreDiagnosticsWatchdog.WaitStatisticsAvailableStatus, StringComparison.Ordinal)) + { + Assert.NotNull(slowQuery.TotalWaitMilliseconds); + Assert.NotNull(slowQuery.AverageWaitMilliseconds); + Assert.False(string.IsNullOrEmpty(slowQuery.TopWaitCategory)); + } + else + { + Assert.Null(slowQuery.TotalWaitMilliseconds); + } + + QueryPlanNotification queryPlan = Assert.Single( + notifications.OfType().ToList(), + notification => notification.QueryId == slowQuery.QueryId && notification.PlanId == slowQuery.PlanId); + Assert.Equal(QueryPlanSanitizer.SanitizedStatus, queryPlan.SanitizationStatus); + Assert.NotNull(queryPlan.SanitizedQueryPlan); } - QueryPlanNotification queryPlan = Assert.Single( - notifications.FindAll(notification => notification is QueryPlanNotification) - .ConvertAll(notification => (QueryPlanNotification)notification) - .FindAll(notification => notification.QueryId == slowQuery.QueryId && notification.PlanId == slowQuery.PlanId)); - Assert.Equal(QueryPlanSanitizer.SanitizedStatus, queryPlan.SanitizationStatus); - Assert.NotNull(queryPlan.SanitizedQueryPlan); + AssertStatisticsHealthOrdinals(notifications, tableName); - Assert.NotEmpty(notifications.FindAll(notification => notification is StatisticsHealthNotification)); - - foreach (IMetricsNotification notification in notifications.FindAll(notification => notification is SlowQueryNotification)) + foreach (SlowQueryNotification slowQuery in notifications.OfType()) { - string queryText = ((SlowQueryNotification)notification).QueryText; - Assert.DoesNotContain("query_store", queryText, StringComparison.OrdinalIgnoreCase); - Assert.DoesNotContain("dm_db_stats_properties", queryText, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("query_store", slowQuery.QueryText, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("dm_db_stats_properties", slowQuery.QueryText, StringComparison.OrdinalIgnoreCase); } } finally @@ -175,6 +197,68 @@ public async Task GivenRuntimeGateDisabled_WhenRun_ThenPublishesNothing() } } + private static void AssertStatisticsHealthOrdinals(List notifications, string probeTableName) + { + // Every column of the statistics-health read is taken positionally from a hand-written SELECT list that + // no compiler checks, so two same-typed columns could be reordered and every value would silently swap. + // The probe table is set up so that its statistics carry values that differ from one another, which pins + // those ordinals: a swap would have to preserve every one of these values to go unnoticed. + List statisticsHealth = notifications.OfType().ToList(); + Assert.NotEmpty(statisticsHealth); + + StatisticsHealthNotification probeIndexStatistics = Assert.Single( + statisticsHealth, + notification => + string.Equals(notification.SchemaName, "dbo", StringComparison.Ordinal) + && string.Equals(notification.TableName, probeTableName, StringComparison.Ordinal) + && string.Equals(notification.StatisticsName, $"PK_{probeTableName}", StringComparison.Ordinal)); + + // The probe table holds ProbeTableRowCount rows at the point its statistics were updated with a full + // scan, and exactly ProbeTableModificationCount rows were added afterwards, so every numeric column has + // a known and distinct value rather than a coincidentally equal one. + Assert.NotNull(probeIndexStatistics.LastUpdated); + Assert.NotNull(probeIndexStatistics.Rows); + Assert.NotNull(probeIndexStatistics.RowsSampled); + Assert.NotNull(probeIndexStatistics.ModificationCounter); + Assert.Equal((long)ProbeTableRowCount, probeIndexStatistics.Rows.Value); + Assert.Equal((long)ProbeTableRowCount, probeIndexStatistics.RowsSampled.Value); + Assert.Equal((long)ProbeTableModificationCount, probeIndexStatistics.ModificationCounter.Value); + Assert.NotNull(probeIndexStatistics.ModificationPercent); + Assert.Equal(ProbeTableModificationCount * 100.0 / ProbeTableRowCount, probeIndexStatistics.ModificationPercent.Value, 6); + + // Statistics backed by an index report is_from_index, and nothing else. + Assert.True(probeIndexStatistics.IsFromIndex); + Assert.False(probeIndexStatistics.IsAutoCreated); + Assert.False(probeIndexStatistics.IsUserCreated); + Assert.False(probeIndexStatistics.HasFilter); + + // A standalone CREATE STATISTICS object reports user_created, and nothing else, which separates that + // flag from the three bit columns adjacent to it. + StatisticsHealthNotification probeUserStatistics = Assert.Single( + statisticsHealth, + notification => + string.Equals(notification.SchemaName, "dbo", StringComparison.Ordinal) + && string.Equals(notification.TableName, probeTableName, StringComparison.Ordinal) + && string.Equals(notification.StatisticsName, $"ST_{probeTableName}", StringComparison.Ordinal)); + Assert.True(probeUserStatistics.IsUserCreated); + Assert.False(probeUserStatistics.IsFromIndex); + Assert.False(probeUserStatistics.IsAutoCreated); + Assert.False(probeUserStatistics.HasFilter); + + // A real FHIR table is asserted on as well, so that the scan is not merely finding the table this test + // created. This index is filtered, which is what separates has_filter from is_from_index. + StatisticsHealthNotification filteredIndexStatistics = Assert.Single( + statisticsHealth, + notification => + string.Equals(notification.SchemaName, "dbo", StringComparison.Ordinal) + && string.Equals(notification.TableName, "Resource", StringComparison.Ordinal) + && string.Equals(notification.StatisticsName, "IX_Resource_ResourceTypeId_ResourceId", StringComparison.Ordinal)); + Assert.True(filteredIndexStatistics.HasFilter); + Assert.True(filteredIndexStatistics.IsFromIndex); + Assert.False(filteredIndexStatistics.IsAutoCreated); + Assert.False(filteredIndexStatistics.IsUserCreated); + } + private QueryStoreDiagnosticsWatchdog CreateWatchdog(IMediator mediator, bool enabled) { var configuration = new WatchdogConfiguration(); @@ -184,7 +268,10 @@ private QueryStoreDiagnosticsWatchdog CreateWatchdog(IMediator mediator, bool en configuration.QueryStoreDiagnostics.MinDurationMilliseconds = 1; configuration.QueryStoreDiagnostics.IncludeQueryPlans = true; configuration.QueryStoreDiagnostics.IncludeStatisticsHealth = true; - configuration.QueryStoreDiagnostics.StatisticsHealthCount = 50; + + // High enough to cover every statistics object in the FHIR schema, so that the assertions on named + // statistics do not depend on where the staleness ordering happens to place them. + configuration.QueryStoreDiagnostics.StatisticsHealthCount = 5000; return new QueryStoreDiagnosticsWatchdog( _fixture.SqlRetryService, @@ -257,12 +344,28 @@ DELETE FROM dbo.Parameters private static async Task CreateProbeTableAsync(SqlConnection connection, string tableName, CancellationToken cancellationToken) { + // The primary key is named explicitly so that the statistics object it backs has a predictable name to + // assert on; an unnamed constraint would get a generated one. await ExecuteNonQueryAsync( connection, - $"CREATE TABLE dbo.[{tableName}] (Id int NOT NULL PRIMARY KEY); INSERT INTO dbo.[{tableName}] (Id) SELECT TOP (200) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) FROM sys.all_objects;", + $"CREATE TABLE dbo.[{tableName}] (Id int NOT NULL CONSTRAINT [PK_{tableName}] PRIMARY KEY); INSERT INTO dbo.[{tableName}] (Id) SELECT TOP ({ProbeTableRowCount}) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) FROM sys.all_objects;", cancellationToken); } + private static async Task PrepareStatisticsProbeAsync(SqlConnection connection, string tableName, CancellationToken cancellationToken) + { + // A full-scan update fixes rows and rows_sampled at the current row count and sets last_updated; the + // rows inserted afterwards then fix modification_counter at a different, known value. Without this the + // table's statistics have never been updated and dm_db_stats_properties reports nulls throughout, which + // asserts nothing about which column was read. + string commandText = $@" +UPDATE STATISTICS dbo.[{tableName}] WITH FULLSCAN; +CREATE STATISTICS [ST_{tableName}] ON dbo.[{tableName}] (Id) WITH FULLSCAN; +INSERT INTO dbo.[{tableName}] (Id) SELECT TOP ({ProbeTableModificationCount}) {ProbeTableRowCount} + ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) FROM sys.all_objects;"; + + await ExecuteNonQueryAsync(connection, commandText, cancellationToken); + } + private static async Task DropProbeTableAsync(SqlConnection connection, string tableName, CancellationToken cancellationToken) { await ExecuteNonQueryAsync(connection, $"DROP TABLE IF EXISTS dbo.[{tableName}];", cancellationToken); From 90c2add6c32bcd1cc6320b457d891a2bc9cba4a5 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 00:02:59 +0000 Subject: [PATCH 12/20] Surface the silent PeriodSec override and document enablement Reviewing the configuration surface turned up two ways an operator can set something and get no effect and no error. dbo.Parameters declares its primary key WITH (IGNORE_DUP_KEY = ON), so the seeding INSERT in Watchdog.InitParamsAsync is a silent no-op on a database that already holds the row, and the following line reads the stored value back over the configured one. Changing the configured PeriodSec on an initialized database therefore does nothing, silently, for both the tick interval and the lookback window. This is shared framework behaviour, so rather than change it, the watchdog now logs a warning at initialization when the stored period differs from the configured one. The runtime gate has no configuration binding at all: it is seeded to 0 and can only be armed with an UPDATE against dbo.Parameters. Setting only FhirServer:Watchdog:QueryStoreDiagnostics:Enabled collects nothing and logs "is not enabled". The doc now says so and gives the statement, and records that the remaining six settings bind through IOptions and so take effect on restart rather than reloading in place. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 18 +++++++++++++++++- .../Watchdogs/QueryStoreDiagnosticsWatchdog.cs | 14 ++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 0a85043a57..053aa34052 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -63,6 +63,14 @@ The feature is **off by default** and is gated by two independent switches. Both This two-gate arrangement is what makes the feature safe to ship dark: the configuration gate keeps it off for the fleet, and the `dbo.Parameters` gate lets an investigation be turned on for one affected account and turned off again afterwards. +**Setting the configuration gate alone collects nothing.** The runtime gate is seeded to `0` by `InitAdditionalParamsAsync` and has no configuration binding, so it cannot be set through `appsettings.json` or an environment variable — arming it is deliberately a separate, deliberate act against the database: + +```sql +UPDATE dbo.Parameters SET Number = 1 WHERE Id = 'QueryStoreDiagnosticsWatchdog.IsEnabled' +``` + +Until that row is `1` the watchdog logs `QueryStoreDiagnosticsWatchdog is not enabled. Exiting...` once per tick and returns. That log line is the thing to look for when the feature appears to be doing nothing. + ### Configuration `WatchdogConfiguration.QueryStoreDiagnostics`, bound from `FhirServer:Watchdog:QueryStoreDiagnostics`. Note that `Watchdog` is a sibling of `Operations` under `FhirServer`, not nested inside it: @@ -70,7 +78,7 @@ This two-gate arrangement is what makes the feature safe to ship dark: the confi | Setting | Default | Meaning | | --- | --- | --- | | `Enabled` | `false` | Deployment-time gate described above. | -| `PeriodSec` | `3600` | Interval between collections. Seeds `dbo.Parameters`; the live value is read from there. Also used as the Query Store lookback window. | +| `PeriodSec` | `3600` | Interval between collections. **Seeds `dbo.Parameters` once**; thereafter the live value is read from there. Also used as the Query Store lookback window. | | `SlowQueryCount` | `10` | Number of slow plans to report per tick. | | `MinDurationMilliseconds` | `1000` | Minimum weighted average duration for a plan to be reported. | | `IncludeQueryPlans` | `true` | Whether sanitized Showplan XML is emitted. | @@ -79,6 +87,14 @@ This two-gate arrangement is what makes the feature safe to ship dark: the confi The lookback window is the live `PeriodSec` clamped to `[60, 86400]` seconds, so the collection window tracks the collection interval and a misconfigured value cannot request an unbounded scan. +**`PeriodSec` is write-once per database.** `Watchdog.InitParamsAsync` inserts it into `dbo.Parameters`, but that table's primary key is declared `WITH (IGNORE_DUP_KEY = ON)`, so on a database that already holds the row the insert is silently a no-op — and the next line reads the stored value back over the configured one. Changing the configured `PeriodSec` on an already-initialized database therefore has **no effect**; the stored row wins, and it governs both the tick interval and the lookback window. This is shared framework behaviour, not specific to this watchdog. To actually change the period, update the row: + +```sql +UPDATE dbo.Parameters SET Number = 900 WHERE Id = 'QueryStoreDiagnosticsWatchdog.PeriodSec' +``` + +Because this override is silent, the watchdog logs a warning at initialization whenever the stored period differs from the configured one, so a deployment that believes it changed the interval is not left to discover otherwise from collection timestamps. The other six settings are read from configuration on every tick and have no `dbo.Parameters` equivalent, so they take effect on restart. All binding is through `IOptions` rather than `IOptionsMonitor`, so no setting reloads in place. + ## Emitted contracts Three notification types implement `IMetricsNotification`, each reporting `FhirOperation` `query-store-diagnostics` and `ResourceType` `System`. Hosts bind handlers to route them; the OSS repository does not prescribe a sink. diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs index bb92f30980..79ca109f60 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs @@ -233,6 +233,20 @@ protected override async Task InitAdditionalParamsAsync() INSERT INTO dbo.Parameters (Id, Number) SELECT @IsEnabledId, 0"); command.Parameters.AddWithValue("@IsEnabledId", IsEnabledId); await command.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); + + // By the time this hook runs, the base class has already overwritten PeriodSec with the value stored in + // dbo.Parameters. That store is write-once in practice: the seeding INSERT is a silent no-op on a database + // that already holds the row, because dbo.Parameters has IGNORE_DUP_KEY = ON. So a deployment that changes + // the configured period on an existing database gets no effect and no error. Surface the divergence rather + // than leaving it to be inferred from collection timestamps. + if (PeriodSec != _configuration.PeriodSec) + { + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: configured PeriodSec is {ConfiguredPeriodSec} but the stored value in dbo.Parameters is {StoredPeriodSec}, which takes precedence and also sets the lookback window. Update the '{PeriodSecId}' row to change it.", + _configuration.PeriodSec, + PeriodSec, + PeriodSecId); + } } protected override async Task RunWorkAsync(CancellationToken cancellationToken) From 323c0e18f11c10cac1323052e441c2a94188ddb9 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 01:18:42 +0000 Subject: [PATCH 13/20] Harden query store diagnostics period handling and PHI fail-closed path Round-3 review fixes for the Query Store diagnostics watchdog. Resilience: - Reject a non-positive or non-finite configured PeriodSec in the constructor and fall back to the class default with a warning. PeriodSec flows to PeriodicTimer via the shared framework, which throws on such values; that would fault this watchdog's task and WatchdogsBackgroundService cancels the token shared by every watchdog, so an off-by-default diagnostics feature could take the transaction and cleanup watchdogs down with it. - Track the effective configured period separately so a rejected value is not also reported as overridden by the stored dbo.Parameters row. Observability: - Warn when the lookback clamp decouples from the unclamped tick interval, naming the unexamined window or the overlap. - Promote the IsEnabled gate to a warning carrying the remedy statement; the row has no config binding, so reaching it always means an explicit opt-in that was never armed. - Warn on a negative MinDurationMilliseconds. PHI boundary: - Fail a rootless plan document closed. It is the one condition that would otherwise skip removal and satisfy verification, emitting the document verbatim. XDocument.Load makes it unreachable today; a PHI boundary should have no path where sanitization is skipped and verification reports success. Simplification and tests: - Derive Truncated in the constructor rather than at four call sites, and replace the single-use QueryPlanResult DTO with a named tuple. - Add hostile-XML DOCTYPE coverage and QueryStoreDiagnosticsPeriodTests for the guard, both divergence directions, and both clamp directions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 10 +- .../Features/Metrics/SlowQueryNotification.cs | 4 +- .../Watchdogs/QueryPlanSanitizerTests.cs | 21 ++ .../QueryStoreDiagnosticsPeriodTests.cs | 231 ++++++++++++++++++ .../Watchdogs/QueryPlanSanitizationResult.cs | 15 +- .../Features/Watchdogs/QueryPlanSanitizer.cs | 28 ++- .../QueryStoreDiagnosticsWatchdog.cs | 139 +++++++++-- 7 files changed, 404 insertions(+), 44 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 053aa34052..b68bc7328f 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -69,7 +69,7 @@ This two-gate arrangement is what makes the feature safe to ship dark: the confi UPDATE dbo.Parameters SET Number = 1 WHERE Id = 'QueryStoreDiagnosticsWatchdog.IsEnabled' ``` -Until that row is `1` the watchdog logs `QueryStoreDiagnosticsWatchdog is not enabled. Exiting...` once per tick and returns. That log line is the thing to look for when the feature appears to be doing nothing. +Until that row is `1` the watchdog logs, at **warning** level, `QueryStoreDiagnosticsWatchdog is enabled in configuration but not armed in dbo.Parameters, so no diagnostics are being collected. Exiting...` once per tick, with the arming statement above included inline, and returns. It is a warning rather than information because the watchdog is only ever started when the configuration gate is on, so reaching that line always means an opt-in that is having no effect. That log line is the thing to look for when the feature appears to be doing nothing. ### Configuration @@ -78,9 +78,9 @@ Until that row is `1` the watchdog logs `QueryStoreDiagnosticsWatchdog is not en | Setting | Default | Meaning | | --- | --- | --- | | `Enabled` | `false` | Deployment-time gate described above. | -| `PeriodSec` | `3600` | Interval between collections. **Seeds `dbo.Parameters` once**; thereafter the live value is read from there. Also used as the Query Store lookback window. | +| `PeriodSec` | `3600` | Interval between collections. **Seeds `dbo.Parameters` once**; thereafter the live value is read from there. Also used as the Query Store lookback window. A non-positive or non-finite value is rejected with a warning and the `3600` default is used, because the shared watchdog timer would otherwise throw and fault every watchdog in the process. | | `SlowQueryCount` | `10` | Number of slow plans to report per tick. | -| `MinDurationMilliseconds` | `1000` | Minimum weighted average duration for a plan to be reported. | +| `MinDurationMilliseconds` | `1000` | Minimum weighted average duration for a plan to be reported. A negative value is treated as `0` — which makes every query qualify — and warns. | | `IncludeQueryPlans` | `true` | Whether sanitized Showplan XML is emitted. | | `IncludeStatisticsHealth` | `true` | Whether statistics metadata is emitted. | | `StatisticsHealthCount` | `20` | Number of statistics rows to report per tick. | @@ -93,6 +93,10 @@ The lookback window is the live `PeriodSec` clamped to `[60, 86400]` seconds, so UPDATE dbo.Parameters SET Number = 900 WHERE Id = 'QueryStoreDiagnosticsWatchdog.PeriodSec' ``` +**That `UPDATE` only takes full effect after a restart, and the two values diverge until then.** The lookback window is re-read from `dbo.Parameters` at the start of *every* tick, but the tick interval is read once, at initialization, and handed to the timer for the life of the process. So the statement above shortens the lookback immediately while collections keep firing at the old interval — going from `3600` to `900` on a running host means each tick then looks back 15 minutes out of the 60 minutes it covers, leaving 45 minutes of every hour unexamined until the process restarts. Lengthening the value has the mirror effect: consecutive ticks overlap and re-report the same plans. Restart the host after changing the row, or accept the gap in the interim. + +The watchdog also warns on every tick where the stored period had to be clamped into the supported lookback range of `[60, 86400]` seconds, naming the effective window and what the clamp costs, because in that case the interval and the lookback are permanently decoupled — the initialization warning below cannot see it, since the configured and stored values agree. + Because this override is silent, the watchdog logs a warning at initialization whenever the stored period differs from the configured one, so a deployment that believes it changed the interval is not left to discover otherwise from collection timestamps. The other six settings are read from configuration on every tick and have no `dbo.Parameters` equivalent, so they take effect on restart. All binding is through `IOptions` rather than `IOptionsMonitor`, so no setting reloads in place. ## Emitted contracts diff --git a/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs b/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs index 4ca0263089..c71abf3e7a 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs @@ -73,7 +73,9 @@ public class SlowQueryNotification : IMetricsNotification public double? AverageWaitMilliseconds { get; set; } /// - /// Gets or sets the wait category with the greatest observed wait time, when Query Store wait statistics are available. + /// Gets or sets the wait category with the greatest observed wait time. This can be null even when + /// is Available, because Query Store itself reports no category + /// for some waits; Available means a wait row was read for the plan, not that it named a category. /// public string TopWaitCategory { get; set; } diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs index fce7150953..48c85a285a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs @@ -176,6 +176,27 @@ public void GivenMalformedPlanXml_WhenSanitized_ThenReturnsInvalidXmlWithoutThro Assert.Equal(0, result.SanitizedLength); } + [Fact] + public void GivenPlanXmlWithADtdEntityDeclaration_WhenSanitized_ThenTheDtdIsRefusedAndNoXmlIsReturned() + { + // Arrange + // Parsing is configured with DtdProcessing.Prohibit and no resolver, so a Showplan carrying a DTD is + // rejected outright rather than having its entities expanded. The assertion that matters as much as the + // status is that nothing comes back with it: this input never reaches removal or verification, so a + // payload here would be an unsanitized payload. + const string queryPlan = @"]>"; + + // Act + QueryPlanSanitizationResult result = QueryPlanSanitizer.Sanitize(queryPlan, QueryStoreDiagnosticsWatchdog.MaxFieldLength); + + // Assert + Assert.Equal(QueryPlanSanitizer.InvalidXmlStatus, result.Status); + Assert.Null(result.Xml); + Assert.False(result.Truncated); + Assert.Equal(queryPlan.Length, result.OriginalLength); + Assert.Equal(0, result.SanitizedLength); + } + [Fact] public void GivenSanitizedPlanExceedingFieldCap_WhenSanitized_ThenReturnsVerifiedTruncatedXml() { diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs new file mode 100644 index 0000000000..d57e9a6002 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs @@ -0,0 +1,231 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Medino; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.SqlServer.Features.Storage; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +{ + /// + /// Covers the collection period, which is the one setting whose misconfiguration reaches outside this feature: + /// it is handed to the shared watchdog timer, it is silently overridden by dbo.Parameters, and it is + /// clamped independently when it is used as the Query Store lookback window. None of those paths is reachable + /// from the integration tests, which call the collection directly and never run initialization. + /// + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public class QueryStoreDiagnosticsPeriodTests + { + private const double DefaultPeriodSec = 3600; + + [Theory] + [InlineData(0d)] + [InlineData(-1d)] + [InlineData(-3600d)] + [InlineData(double.NaN)] + [InlineData(double.PositiveInfinity)] + public void GivenAnUnusableConfiguredPeriod_WhenConstructed_ThenTheDefaultIsUsedAndTheValueIsReported(double configuredPeriodSec) + { + // Arrange + var logger = new CapturingLogger(); + + // Act + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec); + + // Assert + // The period must never reach PeriodicTimer in this state: it throws there, which faults this watchdog's + // task and causes WatchdogsBackgroundService to cancel the token every other watchdog shares. + Assert.Equal(DefaultPeriodSec, watchdog.PeriodSec); + + string warning = Assert.Single(logger.WarningMessages); + Assert.Contains(Format(configuredPeriodSec), warning, StringComparison.Ordinal); + Assert.Contains(Format(DefaultPeriodSec), warning, StringComparison.Ordinal); + } + + [Theory] + [InlineData(1d)] + [InlineData(900d)] + [InlineData(86400d)] + public void GivenAUsableConfiguredPeriod_WhenConstructed_ThenItIsUsedUnchangedAndNothingIsReported(double configuredPeriodSec) + { + // Arrange + var logger = new CapturingLogger(); + + // Act + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec); + + // Assert + Assert.Equal(configuredPeriodSec, watchdog.PeriodSec); + Assert.Empty(logger.WarningMessages); + } + + [Fact] + public void GivenAStoredPeriodDifferentFromTheConfiguredOne_WhenInitialized_ThenTheSilentOverrideIsReported() + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: DefaultPeriodSec); + + // dbo.Parameters is write-once in practice, so the base class overwrites the configured period with the + // stored one during initialization. This is that overwrite. + watchdog.PeriodSec = 900; + + // Act + watchdog.WarnIfStoredPeriodSecOverridesConfiguration(); + + // Assert + string warning = Assert.Single(logger.WarningMessages); + Assert.Contains(Format(DefaultPeriodSec), warning, StringComparison.Ordinal); + Assert.Contains(Format(900d), warning, StringComparison.Ordinal); + Assert.Contains(watchdog.PeriodSecId, warning, StringComparison.Ordinal); + } + + [Fact] + public void GivenAStoredPeriodEqualToTheConfiguredOne_WhenInitialized_ThenNothingIsReported() + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: 900); + + // Act + watchdog.WarnIfStoredPeriodSecOverridesConfiguration(); + + // Assert + Assert.Empty(logger.WarningMessages); + } + + [Fact] + public void GivenAnUnusableConfiguredPeriod_WhenInitialized_ThenTheSubstitutedDefaultIsNotReportedAsAnOverride() + { + // Arrange + // The rejected value was already reported at construction, and the default that replaced it is what got + // seeded into dbo.Parameters, so reporting it again as an override would contradict the first warning. + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: 0); + logger.Clear(); + + // Act + watchdog.WarnIfStoredPeriodSecOverridesConfiguration(); + + // Assert + Assert.Empty(logger.WarningMessages); + } + + [Fact] + public void GivenAStoredPeriodAboveTheLookbackCap_WhenDerivingTheLookback_ThenTheUnexaminedWindowIsReported() + { + // Arrange + const double storedPeriodSec = 604800; // one week + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: DefaultPeriodSec); + + // Act + double lookbackPeriodSec = watchdog.GetLookbackPeriodSec(storedPeriodSec); + + // Assert + Assert.Equal(86400d, lookbackPeriodSec); + + // The tick interval stays at the stored period, so the difference is a permanent coverage gap and the + // warning has to name it rather than leave it to be worked out from the clamp. + string warning = Assert.Single(logger.WarningMessages); + Assert.Contains(Format(storedPeriodSec), warning, StringComparison.Ordinal); + Assert.Contains(Format(86400d), warning, StringComparison.Ordinal); + Assert.Contains(Format(storedPeriodSec - 86400d), warning, StringComparison.Ordinal); + Assert.Contains(watchdog.PeriodSecId, warning, StringComparison.Ordinal); + } + + [Fact] + public void GivenAStoredPeriodBelowTheLookbackFloor_WhenDerivingTheLookback_ThenTheOverlapIsReported() + { + // Arrange + const double storedPeriodSec = 30; + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: DefaultPeriodSec); + + // Act + double lookbackPeriodSec = watchdog.GetLookbackPeriodSec(storedPeriodSec); + + // Assert + Assert.Equal(60d, lookbackPeriodSec); + + string warning = Assert.Single(logger.WarningMessages); + Assert.Contains(Format(storedPeriodSec), warning, StringComparison.Ordinal); + Assert.Contains(Format(60d), warning, StringComparison.Ordinal); + Assert.Contains("overlap", warning, StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData(60d)] + [InlineData(3600d)] + [InlineData(86400d)] + public void GivenAStoredPeriodWithinTheLookbackRange_WhenDerivingTheLookback_ThenItIsUsedUnchangedAndNothingIsReported(double storedPeriodSec) + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: DefaultPeriodSec); + + // Act + double lookbackPeriodSec = watchdog.GetLookbackPeriodSec(storedPeriodSec); + + // Assert + Assert.Equal(storedPeriodSec, lookbackPeriodSec); + Assert.Empty(logger.WarningMessages); + } + + private static QueryStoreDiagnosticsWatchdog CreateWatchdog(ILogger logger, double configuredPeriodSec) + { + var configuration = new WatchdogConfiguration(); + configuration.QueryStoreDiagnostics.Enabled = true; + configuration.QueryStoreDiagnostics.PeriodSec = configuredPeriodSec; + + return new QueryStoreDiagnosticsWatchdog( + Substitute.For(), + logger, + Substitute.For(), + Options.Create(configuration)); + } + + // The logging infrastructure formats message arguments with the invariant culture, so expected values are + // formatted the same way rather than with whatever culture the test host happens to run under. + private static string Format(double value) => value.ToString(CultureInfo.InvariantCulture); + + /// + /// Records what was logged, at what level, with the arguments already substituted, so a test can assert that + /// an operator is told the values they need rather than merely that some warning was raised. + /// + private sealed class CapturingLogger : ILogger + { + private readonly List<(LogLevel Level, string Message)> _entries = new List<(LogLevel Level, string Message)>(); + + internal IReadOnlyList WarningMessages => + _entries.Where(entry => entry.Level == LogLevel.Warning).Select(entry => entry.Message).ToList(); + + public IDisposable BeginScope(TState state) + where TState : notnull + => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + _entries.Add((logLevel, formatter(state, exception))); + } + + internal void Clear() => _entries.Clear(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs index 6364a7d553..1a1e6a8e61 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs @@ -20,13 +20,17 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs /// internal sealed class QueryPlanSanitizationResult { - private QueryPlanSanitizationResult(string status, string xml, bool truncated, int originalLength, int sanitizedLength) + private QueryPlanSanitizationResult(string status, string xml, int originalLength, int sanitizedLength) { Status = status; Xml = xml; - Truncated = truncated; OriginalLength = originalLength; SanitizedLength = sanitizedLength; + + // Derived here rather than supplied by each factory, so the invariant "truncated exactly when the payload + // is shorter than the document it came from" has one implementation instead of four opportunities to + // contradict it. A failure result carries no payload and is therefore never truncated. + Truncated = xml != null && sanitizedLength > xml.Length; } /// @@ -71,7 +75,6 @@ internal static QueryPlanSanitizationResult Sanitized(string xml, int originalLe return new QueryPlanSanitizationResult( QueryPlanSanitizer.SanitizedStatus, xml, - sanitizedLength > xml.Length, originalLength, sanitizedLength); } @@ -82,7 +85,7 @@ internal static QueryPlanSanitizationResult Sanitized(string xml, int originalLe /// A result with null XML. internal static QueryPlanSanitizationResult PlanXmlUnavailable() { - return new QueryPlanSanitizationResult(QueryPlanSanitizer.PlanXmlUnavailableStatus, null, false, 0, 0); + return new QueryPlanSanitizationResult(QueryPlanSanitizer.PlanXmlUnavailableStatus, null, 0, 0); } /// @@ -94,7 +97,7 @@ internal static QueryPlanSanitizationResult InvalidXml(int originalLength) { EnsureArg.IsGte(originalLength, 0, nameof(originalLength)); - return new QueryPlanSanitizationResult(QueryPlanSanitizer.InvalidXmlStatus, null, false, originalLength, 0); + return new QueryPlanSanitizationResult(QueryPlanSanitizer.InvalidXmlStatus, null, originalLength, 0); } /// @@ -108,7 +111,7 @@ internal static QueryPlanSanitizationResult VerificationFailed(int originalLengt EnsureArg.IsGte(originalLength, 0, nameof(originalLength)); EnsureArg.IsGte(sanitizedLength, 0, nameof(sanitizedLength)); - return new QueryPlanSanitizationResult(QueryPlanSanitizer.VerificationFailedStatus, null, false, originalLength, sanitizedLength); + return new QueryPlanSanitizationResult(QueryPlanSanitizer.VerificationFailedStatus, null, originalLength, sanitizedLength); } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs index 9e953efcb4..640bb5e898 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs @@ -50,18 +50,28 @@ internal static QueryPlanSanitizationResult Sanitize(string queryPlanXml, int ma document = XDocument.Load(xmlReader, LoadOptions.PreserveWhitespace); } - var elements = document.Root?.DescendantsAndSelf() + // A document with no root element is handled here and only here, because it is the one condition that + // would otherwise both skip removal (there is nothing to descend from) and satisfy verification + // (nothing sensitive is found in an empty tree), and so would emit the document verbatim. XDocument.Load + // rejects a rootless document today, which makes this unreachable; a PHI boundary must nevertheless + // have no condition under which sanitization is skipped and verification reports success. + if (document.Root == null) + { + return QueryPlanSanitizationResult.VerificationFailed(queryPlanXml.Length, 0); + } + + var elements = document.Root.DescendantsAndSelf() .Where(element => string.Equals(element.Name.LocalName, "ParameterList", StringComparison.OrdinalIgnoreCase)) .ToList(); - elements?.Remove(); + elements.Remove(); - var attributes = document.Root?.DescendantsAndSelf() + var attributes = document.Root.DescendantsAndSelf() .Attributes() .Where(attribute => string.Equals(attribute.Name.LocalName, "ParameterCompiledValue", StringComparison.OrdinalIgnoreCase) || string.Equals(attribute.Name.LocalName, "ParameterRuntimeValue", StringComparison.OrdinalIgnoreCase)) .ToList(); - attributes?.Remove(); + attributes.Remove(); var sanitizedXml = document.ToString(SaveOptions.DisableFormatting); var sanitizedLength = sanitizedXml.Length; @@ -91,11 +101,11 @@ internal static QueryPlanSanitizationResult Sanitize(string queryPlanXml, int ma private static bool ContainsSensitiveParameterData(XDocument document) { // Matching is by local name and namespace-agnostic, exactly as the removal above, so a Showplan namespace - // change between SQL versions cannot let parameter data pass verification. - return document.Root != null && - document.Root.DescendantsAndSelf().Any(element => - IsSensitiveName(element.Name.LocalName) || - element.Attributes().Any(attribute => IsSensitiveName(attribute.Name.LocalName))); + // change between SQL versions cannot let parameter data pass verification. The root is known non-null: + // Sanitize fails a rootless document closed before reaching removal. + return document.Root.DescendantsAndSelf().Any(element => + IsSensitiveName(element.Name.LocalName) || + element.Attributes().Any(attribute => IsSensitiveName(attribute.Name.LocalName))); } private static bool IsSensitiveName(string localName) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs index 79ca109f60..0986a9c1cf 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs @@ -33,6 +33,15 @@ internal sealed class QueryStoreDiagnosticsWatchdog : WatchdogThe wait query itself failed, so wait fields are missing because collection is broken. internal const string WaitStatisticsFailedStatus = "Failed"; + /// The collection interval used when configuration does not supply a usable one. + private const double DefaultPeriodSec = 3600; + + /// The shortest lookback window a collection is allowed to use. + private const double MinLookbackPeriodSec = 60; + + /// The longest lookback window a collection is allowed to use. + private const double MaxLookbackPeriodSec = 86400; + private const string QueryStoreStateSql = @" -- readonly_reason is a bitmask, and it is int here while neighbouring Query Store columns (query_id, plan_id, -- count_executions, rows, rows_sampled, modification_counter) are bigint. That inconsistency is the trap: reading @@ -191,6 +200,11 @@ ELSE CONVERT(float, statisticsProperties.modification_counter) / statisticsPrope private readonly IMediator _mediator; private readonly ISqlRetryService _sqlRetryService; + // The period this instance actually asked for, which is the configured value unless that value was unusable + // and the default was substituted for it. The divergence check at initialization compares against this rather + // than against configuration, so a rejected value cannot also be reported as overridden by the stored row. + private readonly double _configuredPeriodSec; + public QueryStoreDiagnosticsWatchdog( ISqlRetryService sqlRetryService, ILogger logger, @@ -202,7 +216,29 @@ public QueryStoreDiagnosticsWatchdog( _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _mediator = EnsureArg.IsNotNull(mediator, nameof(mediator)); _configuration = EnsureArg.IsNotNull(watchdogConfiguration?.Value, nameof(watchdogConfiguration)).QueryStoreDiagnostics; - PeriodSec = _configuration.PeriodSec; + + // PeriodSec reaches PeriodicTimer through the shared watchdog framework (Watchdog.ExecuteAsync -> + // FhirTimer.ExecuteAsync -> new PeriodicTimer(TimeSpan.FromSeconds(PeriodSec))), which rejects a + // non-positive period. That rejection would fault this watchdog's task, and WatchdogsBackgroundService + // cancels the token shared by every watchdog as soon as one of their tasks completes — so a single + // mistyped value in an off-by-default diagnostics feature would take the transaction and cleanup + // watchdogs down with it. A diagnostics feature degrades instead of failing the host: keep the class + // default and name the rejected value. Non-finite values are rejected on the same grounds, because + // TimeSpan.FromSeconds rejects them for the same reason and with the same blast radius. + if (_configuration.PeriodSec > 0 && double.IsFinite(_configuration.PeriodSec)) + { + PeriodSec = _configuration.PeriodSec; + } + else + { + PeriodSec = DefaultPeriodSec; + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: configured PeriodSec is {ConfiguredPeriodSec}, which is not a usable collection interval. Falling back to {FallbackPeriodSec} seconds. Configure a positive value to change the interval.", + _configuration.PeriodSec, + DefaultPeriodSec); + } + + _configuredPeriodSec = PeriodSec; } internal QueryStoreDiagnosticsWatchdog() @@ -218,7 +254,7 @@ internal QueryStoreDiagnosticsWatchdog() public override bool AllowRebalance { get; internal set; } = true; - public override double PeriodSec { get; internal set; } = 3600; + public override double PeriodSec { get; internal set; } = DefaultPeriodSec; /// /// Exposes RunWorkAsync for unit testing purposes. @@ -239,16 +275,64 @@ protected override async Task InitAdditionalParamsAsync() // that already holds the row, because dbo.Parameters has IGNORE_DUP_KEY = ON. So a deployment that changes // the configured period on an existing database gets no effect and no error. Surface the divergence rather // than leaving it to be inferred from collection timestamps. - if (PeriodSec != _configuration.PeriodSec) + WarnIfStoredPeriodSecOverridesConfiguration(); + } + + /// + /// Reports the stored dbo.Parameters period silently overriding the one this instance was configured + /// with. Separated from , which cannot run without a live database, + /// so the branch is reachable from unit tests. + /// + internal void WarnIfStoredPeriodSecOverridesConfiguration() + { + // Exact comparison is correct here and an epsilon would not be: dbo.Parameters.Number is SQL float, which + // is IEEE-754 double, so a value that originated as a double round-trips losslessly. + if (PeriodSec != _configuredPeriodSec) { _logger.LogWarning( "QueryStoreDiagnosticsWatchdog: configured PeriodSec is {ConfiguredPeriodSec} but the stored value in dbo.Parameters is {StoredPeriodSec}, which takes precedence and also sets the lookback window. Update the '{PeriodSecId}' row to change it.", - _configuration.PeriodSec, + _configuredPeriodSec, PeriodSec, PeriodSecId); } } + /// + /// Clamps the stored collection period into the supported lookback range, reporting what the clamp costs when + /// it changes the value. Exposed as internal only for unit testing. + /// + /// The collection period as stored in dbo.Parameters. + /// The lookback window, in seconds, to use for this collection. + internal double GetLookbackPeriodSec(double storedPeriodSec) + { + var lookbackPeriodSec = Math.Clamp(storedPeriodSec, MinLookbackPeriodSec, MaxLookbackPeriodSec); + + // The tick interval is the stored period unclamped, and it is fixed once at initialization, so whenever the + // clamp bites the two decouple permanently and silently. The initialization warning does not cover this + // case: there the configured and stored values agree, so it stays quiet. + if (lookbackPeriodSec < storedPeriodSec) + { + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: the stored PeriodSec of {StoredPeriodSec} seconds exceeds the maximum lookback window, so every collection looks back only {LookbackPeriodSec} seconds and {UnexaminedPeriodSec} seconds of each interval are never examined. Set the '{PeriodSecId}' row to at most {MaxLookbackPeriodSec} seconds.", + storedPeriodSec, + lookbackPeriodSec, + storedPeriodSec - lookbackPeriodSec, + PeriodSecId, + MaxLookbackPeriodSec); + } + else if (lookbackPeriodSec > storedPeriodSec) + { + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: the stored PeriodSec of {StoredPeriodSec} seconds is below the minimum lookback window, so every collection looks back {LookbackPeriodSec} seconds and consecutive collections overlap and re-report the same plans. Set the '{PeriodSecId}' row to at least {MinLookbackPeriodSec} seconds.", + storedPeriodSec, + lookbackPeriodSec, + PeriodSecId, + MinLookbackPeriodSec); + } + + return lookbackPeriodSec; + } + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { try @@ -261,11 +345,17 @@ protected override async Task RunWorkAsync(CancellationToken cancellationToken) if (!await IsEnabledAsync(cancellationToken)) { - _logger.LogInformation("QueryStoreDiagnosticsWatchdog is not enabled. Exiting..."); + // Warning, not information: WatchdogsBackgroundService only starts this watchdog when the feature + // is enabled in configuration, so reaching this line always means an operator opted in and the + // opt-in is having no effect because the runtime row was never armed. The remedy is inline so it + // does not have to be looked up. + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog is enabled in configuration but not armed in dbo.Parameters, so no diagnostics are being collected. Exiting... Arm it with: UPDATE dbo.Parameters SET Number = 1 WHERE Id = '{IsEnabledId}'", + IsEnabledId); return; } - var lookbackPeriodSec = Math.Clamp(await GetNumberParameterByIdAsync(PeriodSecId, cancellationToken), 60d, 86400d); + var lookbackPeriodSec = GetLookbackPeriodSec(await GetNumberParameterByIdAsync(PeriodSecId, cancellationToken)); var collectionTime = DateTimeOffset.UtcNow; var startTime = collectionTime.AddSeconds(-lookbackPeriodSec); var queryStoreState = await GetQueryStoreStateAsync(cancellationToken); @@ -431,9 +521,20 @@ private async Task GetQueryStoreStateAsync(CancellationToken ca private async Task> GetSlowQueriesAsync(DateTimeOffset startTime, CancellationToken cancellationToken) { await using var command = new SqlCommand(SlowQueriesSql); + var minDurationMilliseconds = Math.Max(0, _configuration.MinDurationMilliseconds); + if (_configuration.MinDurationMilliseconds < 0) + { + // Clamping to zero inverts the operator's intent — every query in the window becomes "slow" — so it + // is reported rather than absorbed, as its sibling thresholds already are. + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: MinDurationMilliseconds is {ConfiguredMinDurationMilliseconds}, which is negative and is being treated as {EffectiveMinDurationMilliseconds}, so every query in the collection window qualifies as slow. Configure a non-negative value.", + _configuration.MinDurationMilliseconds, + minDurationMilliseconds); + } + command.Parameters.Add("@StartTime", SqlDbType.DateTimeOffset).Value = startTime; command.Parameters.Add("@Top", SqlDbType.Int).Value = Math.Max(0, _configuration.SlowQueryCount); - command.Parameters.Add("@MinDurationMilliseconds", SqlDbType.Int).Value = Math.Max(0, _configuration.MinDurationMilliseconds); + command.Parameters.Add("@MinDurationMilliseconds", SqlDbType.Int).Value = minDurationMilliseconds; return await _sqlRetryService.ExecuteReaderAsync( command, @@ -504,19 +605,20 @@ private async Task PublishQueryPlansAsync(IReadOnlyList sl var plans = await _sqlRetryService.ExecuteReaderAsync( command, - reader => new QueryPlanResult( - reader.GetInt64(0), - reader.IsDBNull(1) ? null : reader.GetString(1)), + reader => (PlanId: reader.GetInt64(0), QueryPlanXml: reader.IsDBNull(1) ? null : reader.GetString(1)), _logger, "Failed to read Query Store plans", cancellationToken); - var plansById = plans.ToDictionary(plan => plan.PlanId); + + // Keyed by plan id and holding the plan XML itself: a plan with no row and a plan whose row carries NULL + // XML both yield null here, which is exactly what the sanitizer reports as PlanXmlUnavailable. + var plansById = plans.ToDictionary(plan => plan.PlanId, plan => plan.QueryPlanXml); var publishedPlanCount = 0; foreach (var slowQuery in slowQueries) { plansById.TryGetValue(slowQuery.PlanId, out var queryPlan); - var sanitizedPlan = QueryPlanSanitizer.Sanitize(queryPlan?.QueryPlan, MaxFieldLength); + var sanitizedPlan = QueryPlanSanitizer.Sanitize(queryPlan, MaxFieldLength); if (!string.Equals(sanitizedPlan.Status, QueryPlanSanitizer.SanitizedStatus, StringComparison.Ordinal)) { // Without this, systematic sanitizer breakage looks exactly like "plans are simply unavailable" @@ -733,18 +835,5 @@ internal WaitStatistics(long planId, double totalWaitMilliseconds, string topWai internal string TopWaitCategory { get; } } - - private sealed class QueryPlanResult - { - internal QueryPlanResult(long planId, string queryPlan) - { - PlanId = planId; - QueryPlan = queryPlan; - } - - internal long PlanId { get; } - - internal string QueryPlan { get; } - } } } From 495e433fd71f01bd3605670cb44a621f15691527 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 02:30:13 +0000 Subject: [PATCH 14/20] Add an optional run window to the query store diagnostics watchdog RunStartDate and RunEndDate bound the period during which collection happens. Both are DateTimeOffset? defaulting to null, so behaviour is unchanged when neither is set: an unset start collects from the first tick and an unset end collects indefinitely. The start is inclusive and the end exclusive, so adjacent windows tile without overlapping. The window is evaluated after both enablement gates, so a deployment outside its window still gets the warning that the dbo.Parameters runtime gate was never armed. The watchdog keeps ticking past RunEndDate rather than shutting down. Completing or faulting a watchdog task makes WatchdogsBackgroundService cancel the token shared by every watchdog, so an off-by-default diagnostics feature ending its own timer would take the transaction and cleanup watchdogs with it. An hourly clock comparison costs nothing. Window state is logged only on change. At the default hourly period a window opening in a month would otherwise emit ~720 identical skip lines. The state a process starts in is always logged once, so the reason for silence is available immediately after a restart. A start that is not before the end is an empty window and is warned about at initialization, since nothing downstream would report it. When either bound is set the effective window is logged converted to UTC: a value without an explicit offset binds in the host's local timezone, which is invisible in the configured text and rarely intended. Verified against the real configuration type that a DateTimeOffset? binds from an environment variable, that an offset-less value resolves to local time, and that a malformed value is rejected by the binder exactly as the existing typed settings are. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 35 +- .../QueryStoreDiagnosticsConfiguration.cs | 20 ++ .../QueryStoreDiagnosticsRunWindowTests.cs | 318 ++++++++++++++++++ .../QueryStoreDiagnosticsWatchdog.cs | 162 ++++++++- 4 files changed, 533 insertions(+), 2 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index b68bc7328f..d3a5e7029a 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -84,6 +84,8 @@ Until that row is `1` the watchdog logs, at **warning** level, `QueryStoreDiagno | `IncludeQueryPlans` | `true` | Whether sanitized Showplan XML is emitted. | | `IncludeStatisticsHealth` | `true` | Whether statistics metadata is emitted. | | `StatisticsHealthCount` | `20` | Number of statistics rows to report per tick. | +| `RunStartDate` | `null` (unset) | Inclusive start of the optional run window. Ticks before it collect nothing. `null` means no lower bound. | +| `RunEndDate` | `null` (unset) | **Exclusive** end of the optional run window. Ticks at or after it collect nothing. `null` means no upper bound. | The lookback window is the live `PeriodSec` clamped to `[60, 86400]` seconds, so the collection window tracks the collection interval and a misconfigured value cannot request an unbounded scan. @@ -97,7 +99,38 @@ UPDATE dbo.Parameters SET Number = 900 WHERE Id = 'QueryStoreDiagnosticsWatchdog The watchdog also warns on every tick where the stored period had to be clamped into the supported lookback range of `[60, 86400]` seconds, naming the effective window and what the clamp costs, because in that case the interval and the lookback are permanently decoupled — the initialization warning below cannot see it, since the configured and stored values agree. -Because this override is silent, the watchdog logs a warning at initialization whenever the stored period differs from the configured one, so a deployment that believes it changed the interval is not left to discover otherwise from collection timestamps. The other six settings are read from configuration on every tick and have no `dbo.Parameters` equivalent, so they take effect on restart. All binding is through `IOptions` rather than `IOptionsMonitor`, so no setting reloads in place. +Because this override is silent, the watchdog logs a warning at initialization whenever the stored period differs from the configured one, so a deployment that believes it changed the interval is not left to discover otherwise from collection timestamps. The other eight settings are read from configuration on every tick and have no `dbo.Parameters` equivalent, so they take effect on restart. All binding is through `IOptions` rather than `IOptionsMonitor`, so no setting reloads in place. + +### Run window + +`RunStartDate` and `RunEndDate` bound the period during which collection happens. Both are `DateTimeOffset?` and both default to `null`; the feature behaves exactly as before when neither is set. A tick collects when: + +``` +(RunStartDate == null || now >= RunStartDate) && (RunEndDate == null || now < RunEndDate) +``` + +- **`RunStartDate` is inclusive, `RunEndDate` is exclusive.** The instant that equals the end is already outside the window, so adjacent windows tile without overlapping. +- **`null` means unbounded**, not "now": an unset start collects from the first tick, an unset end collects indefinitely. +- **A start that is not before the end is an empty window** — including a start exactly equal to the end — and nothing will ever be collected. That configuration is logged as a warning at initialization naming both values, because nothing downstream will ever complain about it. + +The window is evaluated **after** both enablement gates, so a deployment that is outside its window still gets the warning telling it that the `dbo.Parameters` runtime gate was never armed. + +**Set the offset explicitly.** A value without one — `2026-03-01T00:00:00` — is bound in the **host's local timezone**, which is invisible in the configured text and is rarely what was intended. Use ISO-8601 with a `Z` suffix, `2026-03-01T00:00:00Z`. As an environment variable: + +``` +FhirServer__Watchdog__QueryStoreDiagnostics__RunStartDate=2026-03-01T00:00:00Z +FhirServer__Watchdog__QueryStoreDiagnostics__RunEndDate=2026-03-08T00:00:00Z +``` + +Whenever either bound is set, the watchdog logs the effective window **converted to UTC** at initialization, so an operator who typed a local time without an offset sees the resolved instant immediately rather than waiting for a window that silently opens hours late — or, for a short window, never visibly opens at all. + +A **malformed** value is not silently ignored and does not silently disable the window: configuration binding throws `InvalidOperationException` naming the offending key. This matches every other typed setting in this section — `PeriodSec`, `SlowQueryCount` and `Enabled` all reject an unparseable value the same way — so a date bound introduces no failure mode the existing settings do not already have. An empty value binds as `null`, which is how a bound is removed rather than mistyped. + +**The watchdog keeps ticking after `RunEndDate`; it does not shut itself down.** Outside the window it evaluates one clock comparison and returns. Self-termination is not an option available to it: completing or faulting a watchdog task makes `WatchdogsBackgroundService` cancel the token that **every** watchdog shares, so an off-by-default diagnostics feature ending its own timer would take the transaction and cleanup watchdogs with it. An hourly comparison costs nothing by contrast. + +The window state is logged **only when it changes** — not open yet, open, closed — at information level. At the default hourly period a window that opens in a month would otherwise produce roughly 720 identical skip lines before collecting anything. The state a process starts in is always logged once, so the reason for silence is available immediately after a restart. + +**The window boundaries take effect without a restart, but the configured values do not.** These settings bind through `IOptions`, so the *values* are read once at process start and editing configuration afterwards has no effect until the host restarts — the same as the other non-`PeriodSec` settings. What changes without a restart is the *clock*: a window configured before the host started will open and close on schedule while the process keeps running, because each tick re-evaluates the fixed boundaries against the current time. Changing a boundary on a running host still requires a restart. ## Emitted contracts diff --git a/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs index e03ba1efef..0d1759a8a1 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs @@ -3,6 +3,8 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; + namespace Microsoft.Health.Fhir.Core.Configs { /// @@ -45,5 +47,23 @@ public class QueryStoreDiagnosticsConfiguration /// Gets or sets the maximum number of table statistics rows reported per collection. /// public int StatisticsHealthCount { get; set; } = 20; + + /// + /// Gets or sets the inclusive start of the run window, before which no diagnostics are collected. + /// Null, the default, means there is no lower bound and collection can begin immediately. + /// A value without an explicit UTC offset is interpreted in the host's local timezone, which is rarely + /// what was intended and is not visible in the configured text, so an ISO-8601 value with a Z + /// suffix such as 2026-03-01T00:00:00Z is recommended. + /// + public DateTimeOffset? RunStartDate { get; set; } + + /// + /// Gets or sets the exclusive end of the run window, at and after which no diagnostics are collected. + /// Null, the default, means there is no upper bound and collection continues indefinitely. + /// A value without an explicit UTC offset is interpreted in the host's local timezone, which is rarely + /// what was intended and is not visible in the configured text, so an ISO-8601 value with a Z + /// suffix such as 2026-03-08T00:00:00Z is recommended. + /// + public DateTimeOffset? RunEndDate { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs new file mode 100644 index 0000000000..f8682a6da7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs @@ -0,0 +1,318 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Medino; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.SqlServer.Features.Storage; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +{ + /// + /// Covers the optional run window, which decides whether a tick collects anything at all. Every boundary case is + /// a clock comparison the integration tests cannot reach — they invoke the collection directly, and the window is + /// evaluated above it — and the failure mode of getting one wrong is silence rather than an error, so the window + /// has to be pinned here or not at all. + /// + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public class QueryStoreDiagnosticsRunWindowTests + { + private const string WindowStart = "2026-03-01T00:00:00Z"; + private const string WindowEnd = "2026-03-08T00:00:00Z"; + + [Theory] + [InlineData("0001-01-01T00:00:00Z")] + [InlineData("2026-03-01T00:00:00Z")] + [InlineData("9999-12-31T23:59:59Z")] + public void GivenNoConfiguredWindow_WhenCheckingAnyTime_ThenCollectionProceeds(string utcNow) + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger); + + // Act + bool isWithinRunWindow = watchdog.IsWithinRunWindow(Parse(utcNow)); + + // Assert + // Both bounds unset is the default configuration, so this is the shape the feature has for every + // deployment that never opts into a window. + Assert.True(isWithinRunWindow); + Assert.Empty(logger.WarningMessages); + } + + [Theory] + [InlineData("2026-02-28T23:59:59Z", false)] + [InlineData("2026-03-01T00:00:00Z", true)] // the start bound is inclusive + [InlineData("2026-03-01T00:00:01Z", true)] + [InlineData("9999-12-31T23:59:59Z", true)] + public void GivenOnlyAStartDate_WhenCheckingATime_ThenTheBoundIsInclusiveAndThereIsNoUpperLimit(string utcNow, bool expected) + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, runStartDate: Parse(WindowStart)); + + // Act + bool isWithinRunWindow = watchdog.IsWithinRunWindow(Parse(utcNow)); + + // Assert + Assert.Equal(expected, isWithinRunWindow); + } + + [Theory] + [InlineData("0001-01-01T00:00:00Z", true)] + [InlineData("2026-03-07T23:59:59Z", true)] + [InlineData("2026-03-08T00:00:00Z", false)] // the end bound is exclusive + [InlineData("2026-03-08T00:00:01Z", false)] + public void GivenOnlyAnEndDate_WhenCheckingATime_ThenTheBoundIsExclusiveAndThereIsNoLowerLimit(string utcNow, bool expected) + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, runEndDate: Parse(WindowEnd)); + + // Act + bool isWithinRunWindow = watchdog.IsWithinRunWindow(Parse(utcNow)); + + // Assert + Assert.Equal(expected, isWithinRunWindow); + } + + [Theory] + [InlineData("2026-02-28T23:59:59Z", false)] + [InlineData("2026-03-01T00:00:00Z", true)] + [InlineData("2026-03-04T12:00:00Z", true)] + [InlineData("2026-03-07T23:59:59Z", true)] + [InlineData("2026-03-08T00:00:00Z", false)] + [InlineData("2026-03-09T00:00:00Z", false)] + public void GivenBothDates_WhenCheckingATime_ThenOnlyTheHalfOpenIntervalCollects(string utcNow, bool expected) + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, Parse(WindowStart), Parse(WindowEnd)); + + // Act + bool isWithinRunWindow = watchdog.IsWithinRunWindow(Parse(utcNow)); + + // Assert + Assert.Equal(expected, isWithinRunWindow); + } + + [Theory] + [InlineData(WindowEnd, WindowEnd, "2026-02-28T23:59:59Z")] + [InlineData(WindowEnd, WindowEnd, "2026-03-08T00:00:00Z")] + [InlineData(WindowEnd, WindowEnd, "9999-12-31T23:59:59Z")] + [InlineData(WindowEnd, WindowStart, "2026-02-28T23:59:59Z")] + [InlineData(WindowEnd, WindowStart, "2026-03-04T12:00:00Z")] + [InlineData(WindowEnd, WindowStart, "2026-03-09T00:00:00Z")] + public void GivenAStartDateNotBeforeTheEndDate_WhenCheckingAnyTime_ThenCollectionNeverProceeds(string runStartDate, string runEndDate, string utcNow) + { + // Arrange + // Start equal to end is as empty as start after end, because the end bound is exclusive: there is no + // instant that is both at or after the start and strictly before the end. + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, Parse(runStartDate), Parse(runEndDate)); + + // Act + bool isWithinRunWindow = watchdog.IsWithinRunWindow(Parse(utcNow)); + + // Assert + Assert.False(isWithinRunWindow); + } + + [Fact] + public void GivenABoundWithANonUtcOffset_WhenCheckingATime_ThenItIsComparedAsTheInstantItDenotes() + { + // Arrange + // 05:00+05:00 is midnight UTC. A bound configured without an explicit offset is bound in the host's local + // timezone and arrives here exactly like this, so the comparison has to be on the instant rather than on + // the wall-clock reading, or such a window opens hours early or late. + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog( + logger, + runStartDate: new DateTimeOffset(2026, 3, 1, 5, 0, 0, TimeSpan.FromHours(5))); + + // Act + bool beforeTheInstant = watchdog.IsWithinRunWindow(Parse("2026-02-28T23:59:59Z")); + bool atTheInstant = watchdog.IsWithinRunWindow(Parse("2026-03-01T00:00:00Z")); + + // Assert + Assert.False(beforeTheInstant); + Assert.True(atTheInstant); + } + + [Fact] + public void GivenRepeatedTicksInTheSameState_WhenCheckingTheWindow_ThenOnlyTheTransitionsAreReported() + { + // Arrange + // At the default hourly period a window that opens in a month would otherwise log around 720 identical + // skip lines, which is what makes the skip unreadable rather than informative. + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, Parse(WindowStart), Parse(WindowEnd)); + + // Act + watchdog.IsWithinRunWindow(Parse("2026-02-27T00:00:00Z")); + watchdog.IsWithinRunWindow(Parse("2026-02-28T00:00:00Z")); + watchdog.IsWithinRunWindow(Parse("2026-03-02T00:00:00Z")); + watchdog.IsWithinRunWindow(Parse("2026-03-03T00:00:00Z")); + watchdog.IsWithinRunWindow(Parse("2026-03-09T00:00:00Z")); + watchdog.IsWithinRunWindow(Parse("2026-03-10T00:00:00Z")); + + // Assert + // Three transitions across six ticks: not open yet, open, closed. + Assert.Equal(3, logger.InformationMessages.Count); + Assert.Contains("has not opened yet", logger.InformationMessages[0], StringComparison.Ordinal); + Assert.Contains("within the configured run window", logger.InformationMessages[1], StringComparison.Ordinal); + Assert.Contains("closed", logger.InformationMessages[2], StringComparison.Ordinal); + + // The skip is the feature working as configured, so none of it is a warning. + Assert.Empty(logger.WarningMessages); + } + + [Fact] + public void GivenTheFirstObservedTick_WhenTheWindowIsAlreadyOpen_ThenTheStateIsStillReportedOnce() + { + // Arrange + // The initial state has to log, or a process that starts inside its window reports nothing about the + // window until the moment it closes. + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, Parse(WindowStart), Parse(WindowEnd)); + + // Act + watchdog.IsWithinRunWindow(Parse("2026-03-02T00:00:00Z")); + watchdog.IsWithinRunWindow(Parse("2026-03-03T00:00:00Z")); + + // Assert + string message = Assert.Single(logger.InformationMessages); + Assert.Contains(FormatUtc(Parse(WindowStart)), message, StringComparison.Ordinal); + Assert.Contains(FormatUtc(Parse(WindowEnd)), message, StringComparison.Ordinal); + } + + [Fact] + public void GivenAnEmptyWindow_WhenInitialized_ThenItIsReportedAsCollectingNothingEver() + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, Parse(WindowEnd), Parse(WindowStart)); + + // Act + watchdog.ReportConfiguredRunWindow(); + + // Assert + // Nothing downstream ever complains about this configuration — it simply never collects — so the warning + // has to name both values rather than say the window is invalid. + string warning = Assert.Single(logger.WarningMessages); + Assert.Contains(FormatUtc(Parse(WindowEnd)), warning, StringComparison.Ordinal); + Assert.Contains(FormatUtc(Parse(WindowStart)), warning, StringComparison.Ordinal); + } + + [Fact] + public void GivenAUsableWindow_WhenInitialized_ThenTheEffectiveWindowIsEchoedInUtcAndNothingIsWarned() + { + // Arrange + // A bound typed without an offset resolves against the host's timezone and looks identical in + // configuration either way, so the resolved UTC instants are echoed at startup where the mistake is still + // cheap to correct. + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog( + logger, + new DateTimeOffset(2026, 3, 1, 5, 0, 0, TimeSpan.FromHours(5)), + new DateTimeOffset(2026, 3, 7, 19, 0, 0, TimeSpan.FromHours(-5))); + + // Act + watchdog.ReportConfiguredRunWindow(); + + // Assert + string message = Assert.Single(logger.InformationMessages); + Assert.Contains(FormatUtc(Parse(WindowStart)), message, StringComparison.Ordinal); + Assert.Contains(FormatUtc(Parse(WindowEnd)), message, StringComparison.Ordinal); + Assert.Empty(logger.WarningMessages); + } + + [Fact] + public void GivenNoConfiguredWindow_WhenInitialized_ThenNothingIsReported() + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger); + + // Act + watchdog.ReportConfiguredRunWindow(); + + // Assert + // The default configuration has no window, and reporting an absent one on every host start would be noise. + Assert.Empty(logger.InformationMessages); + Assert.Empty(logger.WarningMessages); + } + + private static QueryStoreDiagnosticsWatchdog CreateWatchdog( + CapturingLogger logger, + DateTimeOffset? runStartDate = null, + DateTimeOffset? runEndDate = null) + { + var configuration = new WatchdogConfiguration(); + configuration.QueryStoreDiagnostics.Enabled = true; + configuration.QueryStoreDiagnostics.RunStartDate = runStartDate; + configuration.QueryStoreDiagnostics.RunEndDate = runEndDate; + + var watchdog = new QueryStoreDiagnosticsWatchdog( + Substitute.For(), + logger, + Substitute.For(), + Options.Create(configuration)); + + // The shared base constructs a WatchdogLease, which logs through this same logger, so construction is not + // silent. Dropping that here keeps every assertion below a statement about what the run window reported. + logger.Clear(); + + return watchdog; + } + + private static DateTimeOffset Parse(string value) => + DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + + // The logging infrastructure formats message arguments with the invariant culture, so expected values are + // formatted the same way rather than with whatever culture the test host happens to run under. + private static string FormatUtc(DateTimeOffset value) => + value.ToUniversalTime().ToString(CultureInfo.InvariantCulture); + + /// + /// Records what was logged, at what level, with the arguments already substituted, so a test can assert that + /// an operator is told the values they need rather than merely that something was logged. + /// + private sealed class CapturingLogger : ILogger + { + private readonly List<(LogLevel Level, string Message)> _entries = new List<(LogLevel Level, string Message)>(); + + internal IReadOnlyList WarningMessages => + _entries.Where(entry => entry.Level == LogLevel.Warning).Select(entry => entry.Message).ToList(); + + internal IReadOnlyList InformationMessages => + _entries.Where(entry => entry.Level == LogLevel.Information).Select(entry => entry.Message).ToList(); + + public IDisposable BeginScope(TState state) + where TState : notnull + => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + _entries.Add((logLevel, formatter(state, exception))); + } + + internal void Clear() => _entries.Clear(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs index 0986a9c1cf..f900ca9b51 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs @@ -205,6 +205,13 @@ ELSE CONVERT(float, statisticsProperties.modification_counter) / statisticsPrope // than against configuration, so a rejected value cannot also be reported as overridden by the stored row. private readonly double _configuredPeriodSec; + // The run-window state observed on the previous tick, used to log only when the state changes. Null until the + // first observation, so that the state a process starts in is always reported once and an operator never has + // to infer "the window has not opened yet" from the absence of collection logs. Ticks of a given watchdog + // instance are sequential — FhirTimer awaits each RunWorkAsync before the next tick — so this needs no + // synchronization. + private RunWindowState? _lastRunWindowState; + public QueryStoreDiagnosticsWatchdog( ISqlRetryService sqlRetryService, ILogger logger, @@ -247,6 +254,23 @@ internal QueryStoreDiagnosticsWatchdog() // this is used to get param names for testing } + /// + /// Where the current time sits relative to the configured run window. Tracked between ticks so that the + /// skip is reported when it starts and stops rather than on every tick: at the default hourly period, a + /// window that opens in a month would otherwise produce around 720 identical lines before collecting anything. + /// + private enum RunWindowState + { + /// The window has a start and the current time has not reached it yet. + BeforeWindow, + + /// The current time is inside the window, so collection proceeds. + InWindow, + + /// The window has an end and the current time has reached or passed it. + AfterWindow, + } + internal string IsEnabledId => $"{Name}.IsEnabled"; // Ten minutes allows the lease to recover promptly without expiring during a diagnostics collection. @@ -276,6 +300,46 @@ protected override async Task InitAdditionalParamsAsync() // the configured period on an existing database gets no effect and no error. Surface the divergence rather // than leaving it to be inferred from collection timestamps. WarnIfStoredPeriodSecOverridesConfiguration(); + + // A run window that was mistyped produces no collection and no error, which is indistinguishable from a + // window that simply has not opened yet. Report it at startup instead of at the moment it fails to take + // effect, which may be weeks away or never. + ReportConfiguredRunWindow(); + } + + /// + /// Reports the configured run window at initialization: a window that can never open as a warning, and any + /// configured window as its effective UTC bounds. Separated from , + /// which cannot run without a live database, so the branches are reachable from unit tests. + /// + internal void ReportConfiguredRunWindow() + { + DateTimeOffset? runStartDate = _configuration.RunStartDate; + DateTimeOffset? runEndDate = _configuration.RunEndDate; + + if (runStartDate.HasValue && runEndDate.HasValue && runStartDate.Value >= runEndDate.Value) + { + // The end bound is exclusive, so a start equal to the end is as empty as a start after it, and neither + // is reachable by any clock value. Nothing downstream will ever complain, so this is the only place + // the mistake can surface. + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: the configured run window is empty because RunStartDate {RunStartDate} is not before RunEndDate {RunEndDate}, so no diagnostics will ever be collected. Set RunStartDate earlier than RunEndDate, or clear one of them.", + runStartDate.Value, + runEndDate.Value); + } + + if (runStartDate.HasValue || runEndDate.HasValue) + { + // Echoed in UTC on purpose: a value configured without an explicit offset is bound in the host's local + // timezone, and the configured text looks identical either way. Resolving it to a UTC instant here + // lets an operator who meant UTC catch the mistake immediately, rather than discovering it when the + // window silently opens hours late or, for a short window, not visibly at all. + _logger.LogInformation( + "QueryStoreDiagnosticsWatchdog: diagnostics collection is limited to a run window. In UTC it starts at {RunStartDateUtc} inclusive and ends at {RunEndDateUtc} exclusive, where an unset bound is unbounded. Current UTC time is {UtcNow}.", + runStartDate?.ToUniversalTime(), + runEndDate?.ToUniversalTime(), + DateTimeOffset.UtcNow); + } } /// @@ -333,6 +397,89 @@ internal double GetLookbackPeriodSec(double storedPeriodSec) return lookbackPeriodSec; } + /// + /// Determines whether the supplied instant falls inside the configured run window, reporting the state only + /// when it changes. The instant is a parameter rather than read from the clock inside, for the same reason + /// takes the stored period as one: it is the only way to reach + /// every branch from a unit test without introducing a mockable clock abstraction for a single comparison. + /// Exposed as internal only for unit testing. + /// + /// The current time, supplied by the caller. + /// True when collection should proceed; false when the window has not opened or has closed. + internal bool IsWithinRunWindow(DateTimeOffset utcNow) + { + DateTimeOffset? runStartDate = _configuration.RunStartDate; + DateTimeOffset? runEndDate = _configuration.RunEndDate; + + // DateTimeOffset comparisons are made on the underlying UTC instant, so a bound configured with any offset + // — including one implied by the host's local timezone — compares correctly against a UTC clock reading + // without converting anything first. + RunWindowState state; + if (runStartDate.HasValue && utcNow < runStartDate.Value) + { + state = RunWindowState.BeforeWindow; + } + else if (runEndDate.HasValue && utcNow >= runEndDate.Value) + { + // The end bound is exclusive, so the instant that equals it is already outside. That also makes + // adjacent windows tile without overlapping. + state = RunWindowState.AfterWindow; + } + else + { + state = RunWindowState.InWindow; + } + + if (_lastRunWindowState != state) + { + _lastRunWindowState = state; + LogRunWindowState(state, runStartDate, runEndDate, utcNow); + } + + return state == RunWindowState.InWindow; + } + + /// + /// Emits the one line that explains why collection is or is not happening, on the tick where the run-window + /// state changed. + /// + /// The state that has just been entered. + /// The configured start bound, or null when unbounded. + /// The configured end bound, or null when unbounded. + /// The instant the state was observed at. + private void LogRunWindowState(RunWindowState state, DateTimeOffset? runStartDate, DateTimeOffset? runEndDate, DateTimeOffset utcNow) + { + // Information rather than warning throughout: a closed window is the feature working as configured, not a + // misconfiguration. Bounds are echoed in UTC so they can be compared against the current time at a glance + // without mentally applying whatever offset they were configured with. + switch (state) + { + case RunWindowState.BeforeWindow: + _logger.LogInformation( + "QueryStoreDiagnosticsWatchdog: the configured run window has not opened yet, so collection is skipped. It opens at {RunStartDateUtc} and the current UTC time is {UtcNow}.", + runStartDate?.ToUniversalTime(), + utcNow); + break; + + case RunWindowState.AfterWindow: + // Says explicitly that ticking continues, because the obvious reading of "the window has closed" + // is that the watchdog stopped, and someone would otherwise go looking for a process that died. + _logger.LogInformation( + "QueryStoreDiagnosticsWatchdog: the configured run window closed at {RunEndDateUtc} and the current UTC time is {UtcNow}, so collection is skipped. The watchdog keeps ticking and will collect again if the window is widened and the host restarted.", + runEndDate?.ToUniversalTime(), + utcNow); + break; + + default: + _logger.LogInformation( + "QueryStoreDiagnosticsWatchdog: collection is within the configured run window, which starts at {RunStartDateUtc} inclusive and ends at {RunEndDateUtc} exclusive in UTC, where an unset bound is unbounded. The current UTC time is {UtcNow}.", + runStartDate?.ToUniversalTime(), + runEndDate?.ToUniversalTime(), + utcNow); + break; + } + } + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { try @@ -355,8 +502,21 @@ protected override async Task RunWorkAsync(CancellationToken cancellationToken) return; } - var lookbackPeriodSec = GetLookbackPeriodSec(await GetNumberParameterByIdAsync(PeriodSecId, cancellationToken)); + // One clock reading serves both the window decision and the collection window, so a tick that is + // admitted cannot then collect for an instant on the other side of its own boundary. var collectionTime = DateTimeOffset.UtcNow; + if (!IsWithinRunWindow(collectionTime)) + { + // Deliberately a skipped tick rather than a shutdown, even once the end date has passed. Stopping + // would mean faulting or completing this watchdog's task, and WatchdogsBackgroundService cancels + // the token shared by EVERY watchdog as soon as one task completes — so self-terminating an + // off-by-default diagnostics feature would take the transaction and cleanup watchdogs down with + // it. PeriodSec and timer faults propagate through the same shared framework for the same reason. + // A clock comparison once an hour costs nothing; the alternative costs the host. + return; + } + + var lookbackPeriodSec = GetLookbackPeriodSec(await GetNumberParameterByIdAsync(PeriodSecId, cancellationToken)); var startTime = collectionTime.AddSeconds(-lookbackPeriodSec); var queryStoreState = await GetQueryStoreStateAsync(cancellationToken); if (queryStoreState == null) From bad2ec23c90a755e02c8de84d3d8bb46aa927756 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Sat, 22 Aug 2026 03:17:19 +0000 Subject: [PATCH 15/20] Configure query store diagnostics only from configuration, and add ADR The watchdog no longer reads or writes dbo.Parameters. Arming the feature previously required an operator UPDATE against that table, and the base class read its period back over the configured value, so configuration was not authoritative. Both are the pattern the team is moving away from: a diagnostics feature should not require writing to the database. Watchdog.InitParamsAsync is private and non-virtual and is called unconditionally from ExecuteAsync, so there was no override hook. This watchdog therefore schedules itself, owning a FhirTimer and a WatchdogLease directly and reproducing the lease-holder gate, the capped randomized stagger and the per-tick timing line. The lease is kept deliberately: it is runtime coordination rather than configuration, and without it every replica would collect and emit the same diagnostics each period. WatchdogLease was constrained to T : Watchdog while using its type argument only for typeof(T).Name. The constraint restricted nothing the class used and is relaxed here so a self-scheduling component can elect a single replica; every existing caller passes a Watchdog and is unaffected. The alternative, an abstract type existing only to satisfy the constraint, would have reintroduced a Watchdog subclass into the feature that exists to leave it. The constructor period guard is retained and still load-bearing: WatchdogsBackgroundService cancels the token shared by every watchdog as soon as one task completes, so a bad period would still take the transaction and cleanup watchdogs down. Adds ADR-2608 recording why collection runs as an in-process job rather than through an external caller granted rights on the data plane, directly or fronted by Geneva Actions. The deciding argument is that the job introduces no new access path: it runs on the identity the server already holds, pushes results through the existing notification pipeline, and is enabled through the existing configuration surface. Integration coverage now asserts against a live database that the watchdog creates no dbo.Parameters rows, and a unit test asserts the type declares no dbo.Parameters literal and derives from object, so re-deriving would fail rather than silently restore the seeding insert. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 64 ++-- ...608-query-store-performance-diagnostics.md | 58 ++++ .../QueryStoreDiagnosticsConfiguration.cs | 7 +- ...yStoreDiagnosticsConfigurationOnlyTests.cs | 58 ++++ .../QueryStoreDiagnosticsPeriodTests.cs | 97 ++---- .../QueryStoreDiagnosticsRunWindowTests.cs | 5 +- .../QueryStoreDiagnosticsWatchdog.cs | 277 ++++++++++-------- .../Features/Watchdogs/WatchdogLease.cs | 5 +- .../QueryStoreDiagnosticsWatchdogTests.cs | 72 ++--- 9 files changed, 377 insertions(+), 266 deletions(-) create mode 100644 docs/arch/adr-2608-query-store-performance-diagnostics.md create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsConfigurationOnlyTests.cs diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index d3a5e7029a..3d6b528c13 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -45,31 +45,27 @@ The critical property is that **nothing new connects inbound to the database**. ### Why a watchdog -`Watchdog` already provides everything this feature needs, and every one of these behaviours would otherwise have to be reinvented: +`WatchdogsBackgroundService` and `WatchdogLease` already provide what this feature needs, and each of these would otherwise have to be reinvented: -- **Single-runner election.** `WatchdogLease` ensures exactly one instance in a multi-instance deployment performs the work, so an eight-instance service does not issue eight concurrent Query Store scans. -- **Runtime-tunable period.** `PeriodSec` is seeded into `dbo.Parameters` on first run and re-read from there, so the interval can be changed on a live database without a redeploy. -- **An established runtime override.** `DefragWatchdog` already uses a `{Name}.IsEnabled` row in `dbo.Parameters` as an operational switch. This feature reuses that idiom. +- **Single-runner election.** `WatchdogLease` ensures exactly one instance in a multi-instance deployment performs the work, so an eight-instance service does not issue eight concurrent Query Store scans and emit eight copies of every notification. The lease is invisible runtime coordination rather than configuration: it holds nothing an operator sets, and it lives in `dbo.WatchdogLeases` behind `dbo.AcquireWatchdogLease`, shared with every other watchdog. +- **A managed background timer.** `FhirTimer` supplies the randomized start-up stagger that keeps replicas from collecting on the same second, and catches whatever a tick throws so that a failed collection costs one tick rather than the process. - **A precedent for emitting SQL telemetry.** `GeoReplicationLagWatchdog` reads a SQL view on a timer and publishes an `IMetricsNotification`; the host binds a handler that forwards it. This feature is the same shape. +**It deliberately does not derive from `Watchdog`.** That base class inserts `{Name}.PeriodSec` and `{Name}.LeasePeriodSec` into `dbo.Parameters` on every start and then reads the period back **over** the configured value, from a private, non-virtual initialization step with no override hook. This feature is configured exclusively from configuration and writes nothing to the database, so it owns a `FhirTimer` and a `WatchdogLease` directly and reproduces the rest of what the base class did — the lease-holder gate, the capped randomized stagger, and the per-tick timing line — in its own `ExecuteAsync` and tick handler. `WatchdogLease` uses its type argument only to derive the lease resource name from `typeof(T).Name`; its former `T : Watchdog` constraint restricted nothing it actually used, so it was relaxed to admit a self-scheduling component. Every other caller passes a `Watchdog` and is unaffected, and the timer and base class are used unmodified. + ### Enablement -The feature is **off by default** and is gated by two independent switches. Both must be true before any Query Store read occurs. +The feature is **off by default** and is gated by one switch, in configuration. | Gate | Location | Purpose | | --- | --- | --- | -| `FhirServer:Watchdog:QueryStoreDiagnostics:Enabled` | Host configuration | Deployment-time gate. When false the watchdog is never started by `WatchdogsBackgroundService`. | -| `QueryStoreDiagnosticsWatchdog.IsEnabled` | `dbo.Parameters` | Runtime gate. Lets a single account be switched on or off against a live database without a redeploy or restart. | - -This two-gate arrangement is what makes the feature safe to ship dark: the configuration gate keeps it off for the fleet, and the `dbo.Parameters` gate lets an investigation be turned on for one affected account and turned off again afterwards. +| `FhirServer:Watchdog:QueryStoreDiagnostics:Enabled` | Host configuration | When false the watchdog is never started by `WatchdogsBackgroundService`, and no Query Store read occurs. | -**Setting the configuration gate alone collects nothing.** The runtime gate is seeded to `0` by `InitAdditionalParamsAsync` and has no configuration binding, so it cannot be set through `appsettings.json` or an environment variable — arming it is deliberately a separate, deliberate act against the database: +**All configuration for this feature lives in configuration, and the feature writes none of it to the database.** There is no row to seed, arm, or update: no `IsEnabled` row, no `PeriodSec` row, no `LeasePeriodSec` row. Turning the feature on, tuning it, and turning it off are configuration changes plus a restart — no `UPDATE` against a live database, and no possibility of a database holding a value that disagrees with the deployment's configuration. -```sql -UPDATE dbo.Parameters SET Number = 1 WHERE Id = 'QueryStoreDiagnosticsWatchdog.IsEnabled' -``` +The watchdog also re-reads `Enabled` at the start of every tick and returns without collecting when it is false. That check cannot be reached with the value false as the host is wired today: `WatchdogsBackgroundService` gates startup on the same configuration snapshot, and `IOptions` does not reload in place. It is kept because the watchdog is registered `AsSelf` and this is the only place the opt-out is enforced at the unit of work, so a future call site that executes a collection directly cannot bypass it. The cost is one boolean read per period. -Until that row is `1` the watchdog logs, at **warning** level, `QueryStoreDiagnosticsWatchdog is enabled in configuration but not armed in dbo.Parameters, so no diagnostics are being collected. Exiting...` once per tick, with the arming statement above included inline, and returns. It is a warning rather than information because the watchdog is only ever started when the configuration gate is on, so reaching that line always means an opt-in that is having no effect. That log line is the thing to look for when the feature appears to be doing nothing. +The only database row this feature causes to exist is its **lease**, in `dbo.WatchdogLeases` through `dbo.AcquireWatchdogLease`. That is what stops every replica collecting and emitting the same diagnostics every period; it holds no configuration and is the same mechanism every other watchdog uses. That shared stored procedure does consult `dbo.Parameters` for the fleet-wide watchdog lease-holder include and exclude patterns, which is noted under [Configuration](#configuration): it is shared framework behaviour that this feature neither sets nor reads for itself. ### Configuration @@ -77,8 +73,8 @@ Until that row is `1` the watchdog logs, at **warning** level, `QueryStoreDiagno | Setting | Default | Meaning | | --- | --- | --- | -| `Enabled` | `false` | Deployment-time gate described above. | -| `PeriodSec` | `3600` | Interval between collections. **Seeds `dbo.Parameters` once**; thereafter the live value is read from there. Also used as the Query Store lookback window. A non-positive or non-finite value is rejected with a warning and the `3600` default is used, because the shared watchdog timer would otherwise throw and fault every watchdog in the process. | +| `Enabled` | `false` | The gate described above. | +| `PeriodSec` | `3600` | Interval between collections, and — clamped to `[60, 86400]` seconds — the Query Store lookback window each collection covers. A non-positive or non-finite value is rejected with a warning and the `3600` default is used, because `PeriodicTimer` would otherwise throw and fault every watchdog in the process. | | `SlowQueryCount` | `10` | Number of slow plans to report per tick. | | `MinDurationMilliseconds` | `1000` | Minimum weighted average duration for a plan to be reported. A negative value is treated as `0` — which makes every query qualify — and warns. | | `IncludeQueryPlans` | `true` | Whether sanitized Showplan XML is emitted. | @@ -87,19 +83,13 @@ Until that row is `1` the watchdog logs, at **warning** level, `QueryStoreDiagno | `RunStartDate` | `null` (unset) | Inclusive start of the optional run window. Ticks before it collect nothing. `null` means no lower bound. | | `RunEndDate` | `null` (unset) | **Exclusive** end of the optional run window. Ticks at or after it collect nothing. `null` means no upper bound. | -The lookback window is the live `PeriodSec` clamped to `[60, 86400]` seconds, so the collection window tracks the collection interval and a misconfigured value cannot request an unbounded scan. +The lookback window is `PeriodSec` clamped to `[60, 86400]` seconds, so the collection window tracks the collection interval and a misconfigured value cannot request an unbounded scan. -**`PeriodSec` is write-once per database.** `Watchdog.InitParamsAsync` inserts it into `dbo.Parameters`, but that table's primary key is declared `WITH (IGNORE_DUP_KEY = ON)`, so on a database that already holds the row the insert is silently a no-op — and the next line reads the stored value back over the configured one. Changing the configured `PeriodSec` on an already-initialized database therefore has **no effect**; the stored row wins, and it governs both the tick interval and the lookback window. This is shared framework behaviour, not specific to this watchdog. To actually change the period, update the row: +The watchdog warns on every tick where the clamp changed the value, naming the effective window and what the clamp costs, and pointing at `FhirServer:Watchdog:QueryStoreDiagnostics:PeriodSec` as the thing to change. The tick interval itself is **not** clamped, so whenever the clamp bites the interval and the window it covers are permanently decoupled: a period above the cap leaves the excess of every interval unexamined, and one below the floor makes consecutive collections overlap and re-report the same plans. -```sql -UPDATE dbo.Parameters SET Number = 900 WHERE Id = 'QueryStoreDiagnosticsWatchdog.PeriodSec' -``` +**Every setting takes effect on restart, on every database.** All binding is through `IOptions` rather than `IOptionsMonitor`, so nothing reloads in place: editing configuration on a running host changes nothing until it restarts. `PeriodSec` is read once, at construction, because it is handed to the timer for the life of the process; the rest are read from configuration on each tick, which makes no observable difference while the values cannot change underneath. None of them is read from or written to the database, so a deployment behaves identically against a database it created a minute ago and one it has been running against for a year — there is no first-run state for its settings to diverge from. -**That `UPDATE` only takes full effect after a restart, and the two values diverge until then.** The lookback window is re-read from `dbo.Parameters` at the start of *every* tick, but the tick interval is read once, at initialization, and handed to the timer for the life of the process. So the statement above shortens the lookback immediately while collections keep firing at the old interval — going from `3600` to `900` on a running host means each tick then looks back 15 minutes out of the 60 minutes it covers, leaving 45 minutes of every hour unexamined until the process restarts. Lengthening the value has the mirror effect: consecutive ticks overlap and re-report the same plans. Restart the host after changing the row, or accept the gap in the interim. - -The watchdog also warns on every tick where the stored period had to be clamped into the supported lookback range of `[60, 86400]` seconds, naming the effective window and what the clamp costs, because in that case the interval and the lookback are permanently decoupled — the initialization warning below cannot see it, since the configured and stored values agree. - -Because this override is silent, the watchdog logs a warning at initialization whenever the stored period differs from the configured one, so a deployment that believes it changed the interval is not left to discover otherwise from collection timestamps. The other eight settings are read from configuration on every tick and have no `dbo.Parameters` equivalent, so they take effect on restart. All binding is through `IOptions` rather than `IOptionsMonitor`, so no setting reloads in place. +One piece of pre-existing database state can still suppress collection, and it belongs to the shared lease rather than to this feature: `dbo.AcquireWatchdogLease` honours `WatchdogLeaseHolderIncludePattern` and `WatchdogLeaseHolderExcludePattern` rows in `dbo.Parameters`, plus the per-watchdog `...For` variants — where the name is `QueryStoreDiagnosticsWatchdog`. A worker excluded by such a row never becomes lease holder, so its ticks skip and nothing is collected. That applies identically to every watchdog in the process and is not something this feature sets, seeds, or reads itself, but it is the one thing to check on a long-lived database when the feature is enabled and silent. ### Run window @@ -111,9 +101,9 @@ Because this override is silent, the watchdog logs a warning at initialization w - **`RunStartDate` is inclusive, `RunEndDate` is exclusive.** The instant that equals the end is already outside the window, so adjacent windows tile without overlapping. - **`null` means unbounded**, not "now": an unset start collects from the first tick, an unset end collects indefinitely. -- **A start that is not before the end is an empty window** — including a start exactly equal to the end — and nothing will ever be collected. That configuration is logged as a warning at initialization naming both values, because nothing downstream will ever complain about it. +- **A start that is not before the end is an empty window** — including a start exactly equal to the end — and nothing will ever be collected. That configuration is logged as a warning once at startup naming both values, because nothing downstream will ever complain about it. -The window is evaluated **after** both enablement gates, so a deployment that is outside its window still gets the warning telling it that the `dbo.Parameters` runtime gate was never armed. +The window is evaluated inside the tick, after the configuration gate and after the lease-holder check, so a host that is outside its window still holds the lease and keeps ticking — it simply collects nothing and says so once, when the state changes. **Set the offset explicitly.** A value without one — `2026-03-01T00:00:00` — is bound in the **host's local timezone**, which is invisible in the configured text and is rarely what was intended. Use ISO-8601 with a `Z` suffix, `2026-03-01T00:00:00Z`. As an environment variable: @@ -122,7 +112,7 @@ FhirServer__Watchdog__QueryStoreDiagnostics__RunStartDate=2026-03-01T00:00:00Z FhirServer__Watchdog__QueryStoreDiagnostics__RunEndDate=2026-03-08T00:00:00Z ``` -Whenever either bound is set, the watchdog logs the effective window **converted to UTC** at initialization, so an operator who typed a local time without an offset sees the resolved instant immediately rather than waiting for a window that silently opens hours late — or, for a short window, never visibly opens at all. +Whenever either bound is set, the watchdog logs the effective window **converted to UTC** once at startup — before the first tick, and not repeated per tick — so an operator who typed a local time without an offset sees the resolved instant immediately rather than waiting for a window that silently opens hours late — or, for a short window, never visibly opens at all. A **malformed** value is not silently ignored and does not silently disable the window: configuration binding throws `InvalidOperationException` naming the offending key. This matches every other typed setting in this section — `PeriodSec`, `SlowQueryCount` and `Enabled` all reject an unparseable value the same way — so a date bound introduces no failure mode the existing settings do not already have. An empty value binds as `null`, which is how a bound is removed rather than mistyped. @@ -130,7 +120,7 @@ A **malformed** value is not silently ignored and does not silently disable the The window state is logged **only when it changes** — not open yet, open, closed — at information level. At the default hourly period a window that opens in a month would otherwise produce roughly 720 identical skip lines before collecting anything. The state a process starts in is always logged once, so the reason for silence is available immediately after a restart. -**The window boundaries take effect without a restart, but the configured values do not.** These settings bind through `IOptions`, so the *values* are read once at process start and editing configuration afterwards has no effect until the host restarts — the same as the other non-`PeriodSec` settings. What changes without a restart is the *clock*: a window configured before the host started will open and close on schedule while the process keeps running, because each tick re-evaluates the fixed boundaries against the current time. Changing a boundary on a running host still requires a restart. +**The window boundaries take effect without a restart, but the configured values do not.** These settings bind through `IOptions`, so the *values* are read once at process start and editing configuration afterwards has no effect until the host restarts — the same as every other setting in this feature. What changes without a restart is the *clock*: a window configured before the host started will open and close on schedule while the process keeps running, because each tick re-evaluates the fixed boundaries against the current time. Changing a boundary on a running host still requires a restart. ## Emitted contracts @@ -204,13 +194,15 @@ Modification percentage is left null when the row count is null or zero rather t The missing-view handler spans the whole collection rather than each individual read, so the aborting read can be the last one, after slow queries and plans have already been published. Its warning is worded to hold in that case too: it reports that collection was aborted and that whatever had already been emitted was still published, rather than claiming that nothing was collected. -That containment covers **per-tick collection only**. `Watchdog.ExecuteAsync` awaits `InitParamsAsync` *before* and *outside* `FhirTimer`'s per-tick catch, so a throw during initialization — seeding the `dbo.Parameters` rows — still faults the watchdog task, and `WatchdogsBackgroundService` cancels the rest. This is a **pre-existing property of the shared watchdog framework**, not something this feature introduces: `DefragWatchdog` initializes with the identical insert pattern. It is documented here rather than worked around, because changing the shared framework is out of scope for a diagnostics feature. +That containment covers **per-tick collection**, which is where all of this feature's own database work happens: `FhirTimer` catches whatever a tick throws and keeps ticking, so a failed collection costs one tick. The lease renewal is the only other database call, and it runs on the lease's own `FhirTimer` with the same per-tick catch. There is no initialization step left to fail outside either catch. Watchdogs that derive from `Watchdog` do have one — `ExecuteAsync` awaits `InitParamsAsync`, which seeds `dbo.Parameters`, *before* and *outside* the per-tick catch, so a throw there faults the watchdog task and `WatchdogsBackgroundService` cancels the rest — and not deriving from it removes that failure mode here along with the writes. + +What remains outside the catch is the period itself: `PeriodicTimer` throws on a non-positive or non-finite interval, and that throw would happen before the first tick. This is why the configured `PeriodSec` is validated at construction and replaced with the default rather than passed through, and why the run-window check skips a tick rather than ending the timer. ### Reading the primary Every diagnostics read binds to the **primary**, not to a read-only replica, even though these are all read-only queries. -Query Store state is primary-scoped: on a secondary the database is read-only, so `sys.database_query_store_options.actual_state_desc` reports `READ_ONLY`, the `READ_WRITE` state gate rejects it, and the tick returns having emitted nothing — silently, forever, with both gates on and no exception raised. Replica routing is also decided per call against a process-global counter, so a single tick could otherwise check state on the primary and read data from a secondary, harvesting plan identifiers on one server and looking them up on another. +Query Store state is primary-scoped: on a secondary the database is read-only, so `sys.database_query_store_options.actual_state_desc` reports `READ_ONLY`, the `READ_WRITE` state gate rejects it, and the tick returns having emitted nothing — silently, forever, with the feature enabled and no exception raised. Replica routing is also decided per call against a process-global counter, so a single tick could otherwise check state on the primary and read data from a secondary, harvesting plan identifiers on one server and looking them up on another. The cost is one collection per period against the primary — hourly by default — which is negligible next to the failure mode it removes. @@ -266,7 +258,7 @@ Nothing here is PaaS-specific, and no PaaS identity, storage account, or rollout - binding notification handlers and routing the emissions to Geneva or Log Analytics; - setting the configuration gate per environment and ring; -- operating the `dbo.Parameters` runtime override during an investigation; +- setting the collection settings, including the run window, for an investigation; - retention, access control, and downstream handling of emitted query text and plans; and - any responder-facing tooling built on top of the emitted stream. @@ -275,9 +267,9 @@ Nothing here is PaaS-specific, and no PaaS identity, storage account, or rollout 1. Merge the OSS change. The feature ships disabled. 2. Bind a handler and configure routing in `fhir-paas`. 3. Enable the configuration gate in a test ring and confirm emission volume and field sizes. -4. Enable per account through the `dbo.Parameters` override during investigations. +4. Enable it for the deployment under investigation through configuration — bounding it with `RunStartDate` and `RunEndDate` when the investigation is time-boxed — and restart that deployment for the change to take effect. -Because there is **no schema change**, there is no migration ordering dependency and no package/schema-version synchronization step. This is the single largest operational simplification relative to the rejected design. +Because there is **no schema change**, there is no migration ordering dependency and no package/schema-version synchronization step. This is the single largest operational simplification relative to the rejected design. Nothing has to be written to a database to turn the feature on, so there is also no per-database state to seed, clean up, or reconcile against configuration. ## Simplifications and deferred work @@ -320,7 +312,7 @@ Secondary benefits of the change: 1. With Query Store enabled and a deliberately slow query executed, a `SlowQueryNotification` is emitted carrying a matching `QueryId`/`PlanId`. 2. A `QueryPlanNotification` is emitted for that plan with status `Sanitized`. 3. `StatisticsHealthNotification` rows are emitted for user tables. -4. The watchdog performs no work when either gate is off. +4. The watchdog performs no work when the configuration gate is off, and starting it creates no `dbo.Parameters` row and reads none. 5. A non-`READ_WRITE` Query Store state is handled without error and without emission. 6. Wait-statistic unavailability degrades to null wait fields while runtime results are still emitted, and `WaitStatisticsStatus` reports which of the three outcomes occurred. 7. The watchdog does not report its own Query Store queries. diff --git a/docs/arch/adr-2608-query-store-performance-diagnostics.md b/docs/arch/adr-2608-query-store-performance-diagnostics.md new file mode 100644 index 0000000000..c0283fdcec --- /dev/null +++ b/docs/arch/adr-2608-query-store-performance-diagnostics.md @@ -0,0 +1,58 @@ +# ADR-2608: Query Store Performance Diagnostics Collection + +Labels: [SQL](https://github.com/microsoft/fhir-server/labels/Area-SQL) + +**Status**: Proposed +**Date**: 2026-08-22 +**Feature**: Query Store performance diagnostics + +## Context + +Diagnosing a slow FHIR service today requires Azure SQL Query Store data — the slowest recent queries, their execution plans, wait statistics, and statistics health. None of that reaches an on-call engineer or an automated SRE agent without a human who holds database credentials connecting to the customer's data plane and querying it by hand. That is slow during an incident and does not scale across a multi-tenant fleet. + +The constraints that shaped the decision are as much operational as technical. The data plane holds PHI, and Query Store plan XML can embed literal parameter values, so anything that moves plans out of the database is a privacy boundary. The team deliberately operates the service without standing database access, so any design that requires a new identity with rights on customer databases is not merely an implementation detail — it is a permission model the organisation would have to review, provision per environment, rotate, and audit for as long as the feature exists. + +## Options Considered + +1. **External caller executing diagnostics stored procedures** — grant an outside identity, such as the SRE agent, a SQL role restricted to a fixed set of diagnostics stored procedures, and let it connect to the customer database directly. *(rejected: creates a new standing access path into the data plane)* +2. **Geneva Actions brokering the same stored procedures** — keep the procedures, but invoke them through Geneva Actions so the agent never holds SQL credentials itself. *(rejected: moves the credential but does not remove the access path)* +3. **In-process background job that emits diagnostics** — the server collects on its own schedule using the identity it already holds, sanitises plans in C#, and publishes the results through the existing metrics notification pipeline. *(chosen)* + +## Decision + +We chose option 3: a watchdog-style background job inside the FHIR server, off by default, enabled and tuned entirely through configuration. + +The deciding argument is that **it introduces no new access path**. The server is already authenticated to its own database and already runs scheduled background work there; diagnostics collection is additional work on an existing connection and an existing identity. Options 1 and 2 both require a principal that can reach the data plane from outside. Option 2 is genuinely better than option 1 — the agent never holds a credential — but Geneva Actions is a layer in front of the access path, not a replacement for it: the role, its grants, and its lifecycle still have to exist. A SQL reviewer on the team made the same point, that an outside force connecting in and running procedures is a permission model we do not otherwise operate. + +Three consequences of that choice reinforced it. Data now leaves by **push through the existing notification pipeline** that hosts already bind to, rather than by an inbound query into the data plane, which keeps the direction of trust unchanged. **Plan sanitisation moves from T-SQL into C#**, where it is unit-testable, namespace-agnostic across SQL versions, and fails closed by verifying its own output before publishing. And enablement uses the **existing configuration surface**, so switching diagnostics on in an environment is an ordinary deployment change rather than a database operation. + +We further decided that **all settings live in configuration and none in `dbo.Parameters`**. The first iteration followed the existing watchdog convention of keeping runtime knobs in that table, which meant an operator had to run an `UPDATE` to arm the feature and meant the period was read from the database over the top of configuration. That reintroduces database writes for a feature whose purpose is to avoid needing database access, and splits the control surface in two. Honouring it required this watchdog to stop deriving from the shared `Watchdog` base class, whose initialisation privately writes its period rows and reads them back; the distributed lease that prevents duplicate collection across replicas was kept. + +## Consequences + +### Benefits + +- No new principal, role, firewall exception, or credential to provision, rotate, or audit; nothing outside the service gains data-plane access. +- Diagnostics are enabled and tuned per environment through normal configuration, including an optional run window, and are off by default. +- The PHI boundary is enforced in C#, is covered by unit tests, and fails closed rather than emitting an unverified plan. +- Collection is single-instance through the existing lease, so notifications are not multiplied by replica count. + +### Adverse effects + +- Diagnostics cannot be pulled on demand. Data appears on the collection period — hourly by default — so an incident is served by data already being collected, not by an engineer asking a question and getting an immediate answer. A run window has to be configured in advance. +- Settings bind through `IOptions`, so changing them on a running host requires a restart. +- This watchdog no longer shares the `Watchdog` base class and therefore re-implements its timer and lease orchestration and will not inherit future improvements to it. That divergence is the cost of removing `dbo.Parameters`, and should be revisited if the base class itself moves to configuration. +- One shared type changed: `WatchdogLease` was constrained to `T : Watchdog` while using its type argument solely for `typeof(T).Name`. The constraint was relaxed so a self-scheduling component can still elect a single replica. It restricted nothing the class used, and every existing caller passes a `Watchdog` and is unaffected, but it is a shared-file change and reviewers should confirm they are comfortable with it. +- Collection depends on Query Store being enabled and in `READ_WRITE` state on the database; otherwise the job reports why it cannot collect and does nothing. + +### Neutral effects + +- The emitted notifications are contracts that hosts bind to; this repository prescribes no sink. +- The lease continues to write to its own table. That is runtime coordination rather than configuration, and is not part of what this decision removed. + +## References + +- PR [#5723](https://github.com/microsoft/fhir-server/pull/5723) +- `docs/QueryStorePerformanceDiagnostics.md` — design and configuration reference +- `docs/arch/adr-2602-database-logging.md` — precedent for diagnostics gathered inside the service +- `docs/arch/adr-2605-metric-emission-rate-limiting.md` — precedent for emission-rate concerns on the metrics pipeline diff --git a/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs index 0d1759a8a1..fc2c8eb2cb 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs @@ -13,13 +13,14 @@ namespace Microsoft.Health.Fhir.Core.Configs public class QueryStoreDiagnosticsConfiguration { /// - /// Gets or sets a value indicating whether the Query Store diagnostics watchdog can run. - /// The database runtime override must also be enabled. + /// Gets or sets a value indicating whether the Query Store diagnostics watchdog runs. This is the only + /// switch: the feature has no database-side enablement and writes no configuration to the database. /// public bool Enabled { get; set; } = false; /// - /// Gets or sets the interval, in seconds, between diagnostics collections. + /// Gets or sets the interval, in seconds, between diagnostics collections. Also used, clamped to + /// [60, 86400], as the Query Store lookback window each collection covers. /// public double PeriodSec { get; set; } = 3600; diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsConfigurationOnlyTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsConfigurationOnlyTests.cs new file mode 100644 index 0000000000..4a4ed079c2 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsConfigurationOnlyTests.cs @@ -0,0 +1,58 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using System.Reflection; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +{ + /// + /// Pins the property the feature is required to have: it is configured entirely through configuration and never + /// reads or writes dbo.Parameters. Both ways that property could be lost are silent — re-deriving from + /// reintroduces the seeding insert without a line of code being written in this + /// feature, and a hand-written statement is only ever exercised against a live database — so neither is caught + /// by the rest of the unit suite. + /// + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public class QueryStoreDiagnosticsConfigurationOnlyTests + { + [Fact] + public void GivenTheWatchdog_WhenItsTypeIsInspected_ThenItDoesNotInheritTheParameterSeedingBaseClass() + { + // Arrange, Act + Type baseType = typeof(QueryStoreDiagnosticsWatchdog).BaseType; + + // Assert + // Watchdog.ExecuteAsync unconditionally awaits a private, non-virtual InitParamsAsync that inserts + // {Name}.PeriodSec and {Name}.LeasePeriodSec into dbo.Parameters and then reads the period back over the + // configured one. There is no hook to suppress it, so not deriving from it is the mechanism by which + // this feature writes nothing, and re-deriving would undo that without touching this feature's code. + Assert.Equal(typeof(object), baseType); + } + + [Fact] + public void GivenEveryStatementTheWatchdogCanIssue_WhenInspected_ThenNoneReadsOrWritesDboParameters() + { + // Arrange + // Every statement this watchdog issues is a const string on the type, so taking all of its string + // literals is a superset of its SQL — the few non-SQL literals caught alongside them cost nothing. + string[] statements = typeof(QueryStoreDiagnosticsWatchdog) + .GetFields(BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public) + .Where(field => field.IsLiteral && field.FieldType == typeof(string)) + .Select(field => (string)field.GetRawConstantValue()) + .ToArray(); + + // Act, Assert + Assert.NotEmpty(statements); + Assert.All(statements, statement => Assert.DoesNotContain("dbo.Parameters", statement, StringComparison.OrdinalIgnoreCase)); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs index d57e9a6002..fe3b4ccfee 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs @@ -22,9 +22,9 @@ namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs { /// /// Covers the collection period, which is the one setting whose misconfiguration reaches outside this feature: - /// it is handed to the shared watchdog timer, it is silently overridden by dbo.Parameters, and it is - /// clamped independently when it is used as the Query Store lookback window. None of those paths is reachable - /// from the integration tests, which call the collection directly and never run initialization. + /// it is handed to the timer this watchdog owns, and it is clamped independently when it is used as the Query + /// Store lookback window. Neither path is reachable from the integration tests, which call the collection + /// directly and never start the timer. /// [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Operations)] @@ -74,115 +74,66 @@ public void GivenAUsableConfiguredPeriod_WhenConstructed_ThenItIsUsedUnchangedAn } [Fact] - public void GivenAStoredPeriodDifferentFromTheConfiguredOne_WhenInitialized_ThenTheSilentOverrideIsReported() + public void GivenAConfiguredPeriodAboveTheLookbackCap_WhenDerivingTheLookback_ThenTheUnexaminedWindowIsReported() { // Arrange + const double configuredPeriodSec = 604800; // one week var logger = new CapturingLogger(); - QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: DefaultPeriodSec); - - // dbo.Parameters is write-once in practice, so the base class overwrites the configured period with the - // stored one during initialization. This is that overwrite. - watchdog.PeriodSec = 900; - - // Act - watchdog.WarnIfStoredPeriodSecOverridesConfiguration(); - - // Assert - string warning = Assert.Single(logger.WarningMessages); - Assert.Contains(Format(DefaultPeriodSec), warning, StringComparison.Ordinal); - Assert.Contains(Format(900d), warning, StringComparison.Ordinal); - Assert.Contains(watchdog.PeriodSecId, warning, StringComparison.Ordinal); - } - - [Fact] - public void GivenAStoredPeriodEqualToTheConfiguredOne_WhenInitialized_ThenNothingIsReported() - { - // Arrange - var logger = new CapturingLogger(); - QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: 900); - - // Act - watchdog.WarnIfStoredPeriodSecOverridesConfiguration(); - - // Assert - Assert.Empty(logger.WarningMessages); - } - - [Fact] - public void GivenAnUnusableConfiguredPeriod_WhenInitialized_ThenTheSubstitutedDefaultIsNotReportedAsAnOverride() - { - // Arrange - // The rejected value was already reported at construction, and the default that replaced it is what got - // seeded into dbo.Parameters, so reporting it again as an override would contradict the first warning. - var logger = new CapturingLogger(); - QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: 0); - logger.Clear(); - - // Act - watchdog.WarnIfStoredPeriodSecOverridesConfiguration(); - - // Assert - Assert.Empty(logger.WarningMessages); - } - - [Fact] - public void GivenAStoredPeriodAboveTheLookbackCap_WhenDerivingTheLookback_ThenTheUnexaminedWindowIsReported() - { - // Arrange - const double storedPeriodSec = 604800; // one week - var logger = new CapturingLogger(); - QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: DefaultPeriodSec); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec); // Act - double lookbackPeriodSec = watchdog.GetLookbackPeriodSec(storedPeriodSec); + double lookbackPeriodSec = watchdog.GetLookbackPeriodSec(watchdog.PeriodSec); // Assert Assert.Equal(86400d, lookbackPeriodSec); - // The tick interval stays at the stored period, so the difference is a permanent coverage gap and the + // The tick interval stays at the configured period, so the difference is a permanent coverage gap and the // warning has to name it rather than leave it to be worked out from the clamp. string warning = Assert.Single(logger.WarningMessages); - Assert.Contains(Format(storedPeriodSec), warning, StringComparison.Ordinal); + Assert.Contains(Format(configuredPeriodSec), warning, StringComparison.Ordinal); Assert.Contains(Format(86400d), warning, StringComparison.Ordinal); - Assert.Contains(Format(storedPeriodSec - 86400d), warning, StringComparison.Ordinal); - Assert.Contains(watchdog.PeriodSecId, warning, StringComparison.Ordinal); + Assert.Contains(Format(configuredPeriodSec - 86400d), warning, StringComparison.Ordinal); + + // The remedy names the configuration key, because there is no longer a database row to update. + Assert.Contains(QueryStoreDiagnosticsWatchdog.PeriodSecConfigurationKey, warning, StringComparison.Ordinal); } [Fact] - public void GivenAStoredPeriodBelowTheLookbackFloor_WhenDerivingTheLookback_ThenTheOverlapIsReported() + public void GivenAConfiguredPeriodBelowTheLookbackFloor_WhenDerivingTheLookback_ThenTheOverlapIsReported() { // Arrange - const double storedPeriodSec = 30; + const double configuredPeriodSec = 30; var logger = new CapturingLogger(); - QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: DefaultPeriodSec); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec); // Act - double lookbackPeriodSec = watchdog.GetLookbackPeriodSec(storedPeriodSec); + double lookbackPeriodSec = watchdog.GetLookbackPeriodSec(watchdog.PeriodSec); // Assert Assert.Equal(60d, lookbackPeriodSec); string warning = Assert.Single(logger.WarningMessages); - Assert.Contains(Format(storedPeriodSec), warning, StringComparison.Ordinal); + Assert.Contains(Format(configuredPeriodSec), warning, StringComparison.Ordinal); Assert.Contains(Format(60d), warning, StringComparison.Ordinal); Assert.Contains("overlap", warning, StringComparison.OrdinalIgnoreCase); + Assert.Contains(QueryStoreDiagnosticsWatchdog.PeriodSecConfigurationKey, warning, StringComparison.Ordinal); } [Theory] [InlineData(60d)] [InlineData(3600d)] [InlineData(86400d)] - public void GivenAStoredPeriodWithinTheLookbackRange_WhenDerivingTheLookback_ThenItIsUsedUnchangedAndNothingIsReported(double storedPeriodSec) + public void GivenAConfiguredPeriodWithinTheLookbackRange_WhenDerivingTheLookback_ThenItIsUsedUnchangedAndNothingIsReported(double configuredPeriodSec) { // Arrange var logger = new CapturingLogger(); - QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec: DefaultPeriodSec); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredPeriodSec); // Act - double lookbackPeriodSec = watchdog.GetLookbackPeriodSec(storedPeriodSec); + double lookbackPeriodSec = watchdog.GetLookbackPeriodSec(watchdog.PeriodSec); // Assert - Assert.Equal(storedPeriodSec, lookbackPeriodSec); + Assert.Equal(configuredPeriodSec, lookbackPeriodSec); Assert.Empty(logger.WarningMessages); } @@ -224,8 +175,6 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except { _entries.Add((logLevel, formatter(state, exception))); } - - internal void Clear() => _entries.Clear(); } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs index f8682a6da7..e4685914b3 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs @@ -272,8 +272,9 @@ private static QueryStoreDiagnosticsWatchdog CreateWatchdog( Substitute.For(), Options.Create(configuration)); - // The shared base constructs a WatchdogLease, which logs through this same logger, so construction is not - // silent. Dropping that here keeps every assertion below a statement about what the run window reported. + // Constructing the watchdog constructs its WatchdogLease, which logs through this same logger, so + // construction is not silent. Dropping that here keeps every assertion below a statement about what the + // run window reported. logger.Clear(); return watchdog; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs index f900ca9b51..ecc9b1f09c 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Data; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -20,10 +21,24 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - internal sealed class QueryStoreDiagnosticsWatchdog : Watchdog + /// + /// Collects Azure SQL Query Store and statistics diagnostics on a timer and publishes them as metrics + /// notifications. Deliberately does not derive from : that base class seeds and then + /// re-reads its period from dbo.Parameters on every start, from a private non-virtual initialization step + /// with no override hook, and this feature is configured exclusively through configuration and must write + /// nothing to the database. The timer and the lease the base class would have supplied are owned directly + /// instead, so the single-collector guarantee is unchanged. + /// + internal sealed class QueryStoreDiagnosticsWatchdog { internal const int MaxFieldLength = 32 * 1024; + /// + /// The configuration key that sets the collection period, named in the warnings that report the period + /// being unusable or clamped so the remedy does not have to be looked up. + /// + internal const string PeriodSecConfigurationKey = "FhirServer:Watchdog:QueryStoreDiagnostics:PeriodSec"; + /// Wait statistics were read for the plan. internal const string WaitStatisticsAvailableStatus = "Available"; @@ -36,6 +51,26 @@ internal sealed class QueryStoreDiagnosticsWatchdog : WatchdogThe collection interval used when configuration does not supply a usable one. private const double DefaultPeriodSec = 3600; + /// + /// The lease renewal interval. Ten minutes lets the lease be picked up promptly after a replica dies without + /// expiring in the middle of a collection. It is an internal coordination knob rather than an operator + /// setting — nothing an operator can observe changes with it — so it is deliberately not on the + /// configuration surface. + /// + private const double LeasePeriodSec = 600; + + /// + /// Whether the lease may be handed to another replica to balance watchdogs across a deployment. Matches what + /// every other watchdog asks for. + /// + private const bool AllowLeaseRebalance = true; + + /// + /// The cap on the randomized start-up delay. A period longer than an hour would otherwise leave a restarted + /// host collecting nothing for most of a period before its first tick. + /// + private const double MaxInitialDelaySec = 3600; + /// The shortest lookback window a collection is allowed to use. private const double MinLookbackPeriodSec = 60; @@ -199,11 +234,8 @@ ELSE CONVERT(float, statisticsProperties.modification_counter) / statisticsPrope private readonly ILogger _logger; private readonly IMediator _mediator; private readonly ISqlRetryService _sqlRetryService; - - // The period this instance actually asked for, which is the configured value unless that value was unusable - // and the default was substituted for it. The divergence check at initialization compares against this rather - // than against configuration, so a rejected value cannot also be reported as overridden by the stored row. - private readonly double _configuredPeriodSec; + private readonly FhirTimer _fhirTimer; + private readonly WatchdogLease _watchdogLease; // The run-window state observed on the previous tick, used to log only when the state changes. Null until the // first observation, so that the state a process starts in is always reported once and an operator never has @@ -212,26 +244,32 @@ ELSE CONVERT(float, statisticsProperties.modification_counter) / statisticsPrope // synchronization. private RunWindowState? _lastRunWindowState; + // When the per-tick duration was last reported at information level, so that a short period cannot turn that + // line into noise. Written only from the tick, which FhirTimer runs sequentially. + private DateTime _lastTickReported; + public QueryStoreDiagnosticsWatchdog( ISqlRetryService sqlRetryService, ILogger logger, IMediator mediator, IOptions watchdogConfiguration) - : base(sqlRetryService, logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _mediator = EnsureArg.IsNotNull(mediator, nameof(mediator)); _configuration = EnsureArg.IsNotNull(watchdogConfiguration?.Value, nameof(watchdogConfiguration)).QueryStoreDiagnostics; + _fhirTimer = new FhirTimer(_logger); + _watchdogLease = new WatchdogLease(_sqlRetryService, _logger); - // PeriodSec reaches PeriodicTimer through the shared watchdog framework (Watchdog.ExecuteAsync -> + // PeriodSec reaches PeriodicTimer through the timer this watchdog now owns (ExecuteAsync -> // FhirTimer.ExecuteAsync -> new PeriodicTimer(TimeSpan.FromSeconds(PeriodSec))), which rejects a - // non-positive period. That rejection would fault this watchdog's task, and WatchdogsBackgroundService - // cancels the token shared by every watchdog as soon as one of their tasks completes — so a single - // mistyped value in an off-by-default diagnostics feature would take the transaction and cleanup - // watchdogs down with it. A diagnostics feature degrades instead of failing the host: keep the class - // default and name the rejected value. Non-finite values are rejected on the same grounds, because - // TimeSpan.FromSeconds rejects them for the same reason and with the same blast radius. + // non-positive period. Owning the timer does not contain that rejection: it would fault this watchdog's + // task, and WatchdogsBackgroundService cancels the token shared by every watchdog as soon as one of + // their tasks completes — so a single mistyped value in an off-by-default diagnostics feature would + // still take the transaction and cleanup watchdogs down with it. A diagnostics feature degrades instead + // of failing the host: keep the class default and name the rejected value. Non-finite values are + // rejected on the same grounds, because TimeSpan.FromSeconds rejects them for the same reason and with + // the same blast radius. if (_configuration.PeriodSec > 0 && double.IsFinite(_configuration.PeriodSec)) { PeriodSec = _configuration.PeriodSec; @@ -240,18 +278,11 @@ public QueryStoreDiagnosticsWatchdog( { PeriodSec = DefaultPeriodSec; _logger.LogWarning( - "QueryStoreDiagnosticsWatchdog: configured PeriodSec is {ConfiguredPeriodSec}, which is not a usable collection interval. Falling back to {FallbackPeriodSec} seconds. Configure a positive value to change the interval.", + "QueryStoreDiagnosticsWatchdog: configured PeriodSec is {ConfiguredPeriodSec}, which is not a usable collection interval. Falling back to {FallbackPeriodSec} seconds. Configure a positive value in '{PeriodSecConfigurationKey}' to change the interval.", _configuration.PeriodSec, - DefaultPeriodSec); + DefaultPeriodSec, + PeriodSecConfigurationKey); } - - _configuredPeriodSec = PeriodSec; - } - - internal QueryStoreDiagnosticsWatchdog() - : base() - { - // this is used to get param names for testing } /// @@ -271,14 +302,18 @@ private enum RunWindowState AfterWindow, } - internal string IsEnabledId => $"{Name}.IsEnabled"; - - // Ten minutes allows the lease to recover promptly without expiring during a diagnostics collection. - public override double LeasePeriodSec { get; internal set; } = 600; - - public override bool AllowRebalance { get; internal set; } = true; + /// + /// Gets the name this watchdog reports itself under in logs and in the lease it takes. Held as a literal + /// rather than GetType().Name — identical for a sealed class — so that renaming the type surfaces as + /// a deliberate change to a name that appears in operator-facing logs. + /// + public string Name => nameof(QueryStoreDiagnosticsWatchdog); - public override double PeriodSec { get; internal set; } = DefaultPeriodSec; + /// + /// Gets the interval, in seconds, between collections. Set once from configuration at construction, because + /// that is the only source for it and does not reload in place. + /// + public double PeriodSec { get; } /// /// Exposes RunWorkAsync for unit testing purposes. @@ -287,30 +322,72 @@ private enum RunWindowState /// A task representing the asynchronous operation. internal Task RunWorkForTestingAsync(CancellationToken cancellationToken) => RunWorkAsync(cancellationToken); - protected override async Task InitAdditionalParamsAsync() + /// + /// Runs the collection timer and the lease until the supplied token is cancelled. Called by + /// , which only starts this watchdog when the feature is enabled in + /// configuration. + /// + /// The cancellation token. + /// A task representing the asynchronous operation. + public async Task ExecuteAsync(CancellationToken cancellationToken) { - await using var command = new SqlCommand(@" -INSERT INTO dbo.Parameters (Id, Number) SELECT @IsEnabledId, 0"); - command.Parameters.AddWithValue("@IsEnabledId", IsEnabledId); - await command.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); - - // By the time this hook runs, the base class has already overwritten PeriodSec with the value stored in - // dbo.Parameters. That store is write-once in practice: the seeding INSERT is a silent no-op on a database - // that already holds the row, because dbo.Parameters has IGNORE_DUP_KEY = ON. So a deployment that changes - // the configured period on an existing database gets no effect and no error. Surface the divergence rather - // than leaving it to be inferred from collection timestamps. - WarnIfStoredPeriodSecOverridesConfiguration(); - - // A run window that was mistyped produces no collection and no error, which is indistinguishable from a - // window that simply has not opened yet. Report it at startup instead of at the moment it fails to take - // effect, which may be weeks away or never. + _logger.LogDebug("{WatchdogName}.ExecuteAsync: starting...", Name); + + // Reported once per process rather than once per tick. A mistyped window collects nothing and raises + // nothing, which is indistinguishable from a window that has simply not opened yet, so it has to be + // stated at startup — repeating it hourly for the weeks until the window was meant to open would bury + // it. Nothing here touches the database, so it needs no initialization step to hang off. ReportConfiguredRunWindow(); + + // The timer and the lease run concurrently and neither returns until the token is cancelled. The initial + // delay is randomized up to one period and capped at an hour so that replicas started together do not + // all collect on the same second, and so that a long period does not leave a restarted host silent for + // most of it. + await Task.WhenAll( + _fhirTimer.ExecuteAsync(Name, PeriodSec, OnNextTickAsync, cancellationToken, PeriodSec > MaxInitialDelaySec ? MaxInitialDelaySec : PeriodSec), + _watchdogLease.ExecuteAsync($"{Name}Lease", AllowLeaseRebalance, LeasePeriodSec, cancellationToken)); + + _logger.LogDebug("{WatchdogName}.ExecuteAsync: completed.", Name); } /// - /// Reports the configured run window at initialization: a window that can never open as a warning, and any - /// configured window as its effective UTC bounds. Separated from , - /// which cannot run without a live database, so the branches are reachable from unit tests. + /// Runs one tick, on the replica that holds the lease. + /// + /// The cancellation token. + /// A task representing the asynchronous operation. + private async Task OnNextTickAsync(CancellationToken cancellationToken) + { + if (!_watchdogLease.IsLeaseHolder) + { + // The lease is what keeps one collection per period rather than one per replica: without this gate + // an eight-instance deployment would issue eight concurrent Query Store scans an hour and emit + // eight copies of every notification. + _logger.LogDebug("{WatchdogName}.OnNextTickAsync: skipping because this instance does not hold the lease.", Name); + return; + } + + var stopwatch = Stopwatch.StartNew(); + + await RunWorkAsync(cancellationToken); + + // Reports that a tick happened at all, which the collection summary inside RunWorkAsync cannot: a tick + // that returned early — outside the run window, or with Query Store unavailable — logs its reason but + // nothing about the timer still being alive. Throttled to hourly at information level so that a short + // configured period cannot turn it into noise. + if (DateTime.UtcNow - _lastTickReported > TimeSpan.FromHours(1)) + { + _lastTickReported = DateTime.UtcNow; + _logger.LogInformation("{WatchdogName}.OnNextTickAsync ran in {ElapsedMilliseconds} ms.", Name, stopwatch.ElapsedMilliseconds); + } + else + { + _logger.LogDebug("{WatchdogName}.OnNextTickAsync ran in {ElapsedMilliseconds} ms.", Name, stopwatch.ElapsedMilliseconds); + } + } + + /// + /// Reports the configured run window at startup: a window that can never open as a warning, and any + /// configured window as its effective UTC bounds. Exposed as internal only for unit testing. /// internal void ReportConfiguredRunWindow() { @@ -343,54 +420,37 @@ internal void ReportConfiguredRunWindow() } /// - /// Reports the stored dbo.Parameters period silently overriding the one this instance was configured - /// with. Separated from , which cannot run without a live database, - /// so the branch is reachable from unit tests. - /// - internal void WarnIfStoredPeriodSecOverridesConfiguration() - { - // Exact comparison is correct here and an epsilon would not be: dbo.Parameters.Number is SQL float, which - // is IEEE-754 double, so a value that originated as a double round-trips losslessly. - if (PeriodSec != _configuredPeriodSec) - { - _logger.LogWarning( - "QueryStoreDiagnosticsWatchdog: configured PeriodSec is {ConfiguredPeriodSec} but the stored value in dbo.Parameters is {StoredPeriodSec}, which takes precedence and also sets the lookback window. Update the '{PeriodSecId}' row to change it.", - _configuredPeriodSec, - PeriodSec, - PeriodSecId); - } - } - - /// - /// Clamps the stored collection period into the supported lookback range, reporting what the clamp costs when - /// it changes the value. Exposed as internal only for unit testing. + /// Clamps the configured collection period into the supported lookback range, reporting what the clamp costs + /// when it changes the value. Takes the period as a parameter rather than reading + /// directly so that both clamp directions are reachable from a unit test. + /// Exposed as internal only for unit testing. /// - /// The collection period as stored in dbo.Parameters. + /// The collection period this instance is running at. /// The lookback window, in seconds, to use for this collection. - internal double GetLookbackPeriodSec(double storedPeriodSec) + internal double GetLookbackPeriodSec(double configuredPeriodSec) { - var lookbackPeriodSec = Math.Clamp(storedPeriodSec, MinLookbackPeriodSec, MaxLookbackPeriodSec); + var lookbackPeriodSec = Math.Clamp(configuredPeriodSec, MinLookbackPeriodSec, MaxLookbackPeriodSec); - // The tick interval is the stored period unclamped, and it is fixed once at initialization, so whenever the - // clamp bites the two decouple permanently and silently. The initialization warning does not cover this - // case: there the configured and stored values agree, so it stays quiet. - if (lookbackPeriodSec < storedPeriodSec) + // The tick interval is the configured period unclamped, so whenever the clamp bites the interval and the + // window it covers decouple permanently and silently — every tick after the first inherits the same gap + // or the same overlap. + if (lookbackPeriodSec < configuredPeriodSec) { _logger.LogWarning( - "QueryStoreDiagnosticsWatchdog: the stored PeriodSec of {StoredPeriodSec} seconds exceeds the maximum lookback window, so every collection looks back only {LookbackPeriodSec} seconds and {UnexaminedPeriodSec} seconds of each interval are never examined. Set the '{PeriodSecId}' row to at most {MaxLookbackPeriodSec} seconds.", - storedPeriodSec, + "QueryStoreDiagnosticsWatchdog: the configured PeriodSec of {ConfiguredPeriodSec} seconds exceeds the maximum lookback window, so every collection looks back only {LookbackPeriodSec} seconds and {UnexaminedPeriodSec} seconds of each interval are never examined. Set '{PeriodSecConfigurationKey}' to at most {MaxLookbackPeriodSec} seconds.", + configuredPeriodSec, lookbackPeriodSec, - storedPeriodSec - lookbackPeriodSec, - PeriodSecId, + configuredPeriodSec - lookbackPeriodSec, + PeriodSecConfigurationKey, MaxLookbackPeriodSec); } - else if (lookbackPeriodSec > storedPeriodSec) + else if (lookbackPeriodSec > configuredPeriodSec) { _logger.LogWarning( - "QueryStoreDiagnosticsWatchdog: the stored PeriodSec of {StoredPeriodSec} seconds is below the minimum lookback window, so every collection looks back {LookbackPeriodSec} seconds and consecutive collections overlap and re-report the same plans. Set the '{PeriodSecId}' row to at least {MinLookbackPeriodSec} seconds.", - storedPeriodSec, + "QueryStoreDiagnosticsWatchdog: the configured PeriodSec of {ConfiguredPeriodSec} seconds is below the minimum lookback window, so every collection looks back {LookbackPeriodSec} seconds and consecutive collections overlap and re-report the same plans. Set '{PeriodSecConfigurationKey}' to at least {MinLookbackPeriodSec} seconds.", + configuredPeriodSec, lookbackPeriodSec, - PeriodSecId, + PeriodSecConfigurationKey, MinLookbackPeriodSec); } @@ -480,28 +540,22 @@ private void LogRunWindowState(RunWindowState state, DateTimeOffset? runStartDat } } - protected override async Task RunWorkAsync(CancellationToken cancellationToken) + private async Task RunWorkAsync(CancellationToken cancellationToken) { try { if (!_configuration.Enabled) { + // Unreachable in the host as it stands: WatchdogsBackgroundService reads the same configuration + // snapshot and never starts this watchdog when the feature is off, and IOptions does not + // reload in place. It is kept because the watchdog is registered AsSelf and this is the only + // place the opt-out is enforced at the unit of work — any future call site that executes a + // collection directly would otherwise read Query Store on a deployment that opted out. One bool + // read per period is not a cost worth trading that away for. _logger.LogInformation("QueryStoreDiagnosticsWatchdog is disabled by configuration. Exiting..."); return; } - if (!await IsEnabledAsync(cancellationToken)) - { - // Warning, not information: WatchdogsBackgroundService only starts this watchdog when the feature - // is enabled in configuration, so reaching this line always means an operator opted in and the - // opt-in is having no effect because the runtime row was never armed. The remedy is inline so it - // does not have to be looked up. - _logger.LogWarning( - "QueryStoreDiagnosticsWatchdog is enabled in configuration but not armed in dbo.Parameters, so no diagnostics are being collected. Exiting... Arm it with: UPDATE dbo.Parameters SET Number = 1 WHERE Id = '{IsEnabledId}'", - IsEnabledId); - return; - } - // One clock reading serves both the window decision and the collection window, so a tick that is // admitted cannot then collect for an instant on the other side of its own boundary. var collectionTime = DateTimeOffset.UtcNow; @@ -511,12 +565,12 @@ protected override async Task RunWorkAsync(CancellationToken cancellationToken) // would mean faulting or completing this watchdog's task, and WatchdogsBackgroundService cancels // the token shared by EVERY watchdog as soon as one task completes — so self-terminating an // off-by-default diagnostics feature would take the transaction and cleanup watchdogs down with - // it. PeriodSec and timer faults propagate through the same shared framework for the same reason. - // A clock comparison once an hour costs nothing; the alternative costs the host. + // it. PeriodSec and timer faults propagate the same way for the same reason. A clock comparison + // once an hour costs nothing; the alternative costs the host. return; } - var lookbackPeriodSec = GetLookbackPeriodSec(await GetNumberParameterByIdAsync(PeriodSecId, cancellationToken)); + var lookbackPeriodSec = GetLookbackPeriodSec(PeriodSec); var startTime = collectionTime.AddSeconds(-lookbackPeriodSec); var queryStoreState = await GetQueryStoreStateAsync(cancellationToken); if (queryStoreState == null) @@ -542,8 +596,8 @@ protected override async Task RunWorkAsync(CancellationToken cancellationToken) { // This filter covers the whole method body rather than each read, which loses the ability to name the // failing statement. That is the accepted trade-off: the watchdog only runs when an operator enabled - // it in both configuration and dbo.Parameters, so "the views you asked me to read do not exist" is - // always operator-actionable and permanent, and per-read catches would be more churn than value. + // it in configuration, so "the views you asked me to read do not exist" is always operator-actionable + // and permanent, and per-read catches would be more churn than value. // Because the filter spans every read, the missing view can be the last one, after slow queries and // plans have already been published; the message is therefore deliberately worded to be true of a // partial tick as well as of one that emitted nothing. @@ -556,12 +610,9 @@ protected override async Task RunWorkAsync(CancellationToken cancellationToken) } /// - /// Runs the collection itself, once the configuration, runtime and Query Store state gates have all passed. - /// Separated from so that the reads and the notifications they produce are - /// reachable from unit tests: the gates above read dbo.Parameters through - /// , - /// which materializes its value inside a callback executed against a live and so - /// cannot be substituted. Exception handling deliberately stays in the caller. + /// Runs the collection itself, once the configuration, run-window and Query Store state gates have all + /// passed. Separated from so that the reads and the notifications they produce + /// are reachable from unit tests, and so that exception handling stays in one place in the caller. /// /// The start of the collection window. /// The end of the collection window. @@ -850,12 +901,6 @@ private async Task PublishStatisticsHealthAsync(CancellationToken cancellat return statisticsHealth.Count; } - private async Task IsEnabledAsync(CancellationToken cancellationToken) - { - var value = await GetNumberParameterByIdAsync(IsEnabledId, cancellationToken); - return value == 1; - } - /// /// Decodes the sys.database_query_store_options.readonly_reason bitmask into a readable reason list, /// so an operator sees the cause rather than an integer to look up. Exposed as internal only for unit testing. diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs index 4b19944b13..410eb14055 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs @@ -16,8 +16,11 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { + // The type argument supplies the lease resource name through typeof(T).Name and nothing else, so it carries no + // constraint. It deliberately does not require Watchdog: a component that schedules itself, as + // QueryStoreDiagnosticsWatchdog does in order to keep its configuration out of dbo.Parameters, still needs to + // elect a single replica. Every existing caller passes a Watchdog and is unaffected. internal class WatchdogLease - where T : Watchdog { private const double TimeoutFactor = 0.25; private readonly object _locker = new(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs index b72a7cdea0..573bd47cb9 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs @@ -71,7 +71,6 @@ public async Task GivenEnabledWatchdogAndCapturedProbe_WhenRun_ThenPublishesSlow try { await EnableAndVerifyQueryStoreAsync(connection, CancellationToken.None); - await SetWatchdogParametersAsync(connection, isEnabled: 1, periodSeconds: 300, CancellationToken.None); await CreateProbeTableAsync(connection, tableName, CancellationToken.None); double probeElapsedMilliseconds = await ExecuteProbeQueryAsync(connection, tableName, queryAlias, CancellationToken.None); await WaitForQueryStoreCaptureAsync(connection, queryAlias, CancellationToken.None); @@ -155,7 +154,6 @@ public async Task GivenEnabledWatchdogAndCapturedProbe_WhenRun_ThenPublishesSlow finally { await DropProbeTableAsync(connection, tableName, CancellationToken.None); - await DeleteWatchdogParametersAsync(connection, CancellationToken.None); } } @@ -174,27 +172,46 @@ public async Task GivenConfigurationGateDisabled_WhenRun_ThenPublishesNothing() } [Fact] - public async Task GivenRuntimeGateDisabled_WhenRun_ThenPublishesNothing() + public async Task GivenTheWatchdogIsStarted_WhenItRuns_ThenItNeitherSeedsNorReadsAnyDboParametersRow() { // Arrange + // The feature is configured exclusively through configuration, so starting it must leave dbo.Parameters + // untouched. Only a live database can show that: the seeding insert and the period read this watchdog no + // longer performs both happened at startup, before the first tick, and neither is visible to a test that + // invokes the collection directly. var mediator = Substitute.For(); - var watchdog = CreateWatchdog(mediator, enabled: true); + + // A one-second period keeps the randomized start-up delay inside the test's own budget. The lease's first + // acquire attempt is a full lease period away, so no tick of this watchdog reaches a collection here — + // which is the point: what is under test is what running it costs the database before it collects + // anything. + var watchdog = CreateWatchdog(mediator, enabled: true, periodSec: 1); + await using SqlConnection connection = await _fixture.SqlConnectionBuilder.GetSqlConnectionAsync(cancellationToken: CancellationToken.None); await connection.OpenAsync(CancellationToken.None); - await SetWatchdogParametersAsync(connection, isEnabled: 0, periodSeconds: 300, CancellationToken.None); + // A database this test has run the pre-refactor code against still holds the rows it seeded, and they + // would make the assertion below pass for the wrong reason. + await DeleteWatchdogParametersAsync(connection, CancellationToken.None); + + using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + // Act try { - // Act - await watchdog.RunWorkForTestingAsync(CancellationToken.None); - - // Assert - Assert.Empty(mediator.ReceivedCalls()); + await watchdog.ExecuteAsync(cancellationTokenSource.Token); } - finally + catch (OperationCanceledException) { - await DeleteWatchdogParametersAsync(connection, CancellationToken.None); + // Expected whenever the token trips while a randomized start-up delay is still pending, which is the + // usual case. Cancelling between ticks instead returns normally, so neither outcome is asserted on. } + + // Assert + // Had the watchdog seeded anything, the rows would be here. Had it read a period or an enablement flag + // from a row it did not seed, it would have thrown InvalidOperationException out of ExecuteAsync rather + // than being cancelled, because no such row exists. + Assert.Equal(0, await CountWatchdogParametersAsync(connection, CancellationToken.None)); } private static void AssertStatisticsHealthOrdinals(List notifications, string probeTableName) @@ -259,11 +276,11 @@ private static void AssertStatisticsHealthOrdinals(List no Assert.False(filteredIndexStatistics.IsUserCreated); } - private QueryStoreDiagnosticsWatchdog CreateWatchdog(IMediator mediator, bool enabled) + private QueryStoreDiagnosticsWatchdog CreateWatchdog(IMediator mediator, bool enabled, double periodSec = 300) { var configuration = new WatchdogConfiguration(); configuration.QueryStoreDiagnostics.Enabled = enabled; - configuration.QueryStoreDiagnostics.PeriodSec = 300; + configuration.QueryStoreDiagnostics.PeriodSec = periodSec; configuration.QueryStoreDiagnostics.SlowQueryCount = 100; configuration.QueryStoreDiagnostics.MinDurationMilliseconds = 1; configuration.QueryStoreDiagnostics.IncludeQueryPlans = true; @@ -313,33 +330,20 @@ private static async Task GetQueryStoreStateAsync(SqlConnection connecti return (string)await command.ExecuteScalarAsync(cancellationToken); } - private static async Task SetWatchdogParametersAsync(SqlConnection connection, int isEnabled, int periodSeconds, CancellationToken cancellationToken) - { - await SetParameterAsync(connection, "QueryStoreDiagnosticsWatchdog.IsEnabled", isEnabled, cancellationToken); - await SetParameterAsync(connection, "QueryStoreDiagnosticsWatchdog.PeriodSec", periodSeconds, cancellationToken); - } - - private static async Task SetParameterAsync(SqlConnection connection, string id, double value, CancellationToken cancellationToken) + private static async Task DeleteWatchdogParametersAsync(SqlConnection connection, CancellationToken cancellationToken) { await using SqlCommand command = connection.CreateCommand(); - command.CommandText = @" -UPDATE dbo.Parameters SET Number = @Value WHERE Id = @Id; -IF @@ROWCOUNT = 0 -BEGIN - INSERT INTO dbo.Parameters (Id, Number) VALUES (@Id, @Value); -END"; - command.Parameters.AddWithValue("@Id", id); - command.Parameters.AddWithValue("@Value", value); + command.CommandText = "DELETE FROM dbo.Parameters WHERE Id LIKE 'QueryStoreDiagnosticsWatchdog%';"; await command.ExecuteNonQueryAsync(cancellationToken); } - private static async Task DeleteWatchdogParametersAsync(SqlConnection connection, CancellationToken cancellationToken) + private static async Task CountWatchdogParametersAsync(SqlConnection connection, CancellationToken cancellationToken) { + // Matched by prefix rather than by the two names the watchdog used to seed, so a row this feature has no + // business creating is caught whatever it is called. await using SqlCommand command = connection.CreateCommand(); - command.CommandText = @" -DELETE FROM dbo.Parameters -WHERE Id IN ('QueryStoreDiagnosticsWatchdog.IsEnabled', 'QueryStoreDiagnosticsWatchdog.PeriodSec');"; - await command.ExecuteNonQueryAsync(cancellationToken); + command.CommandText = "SELECT COUNT(*) FROM dbo.Parameters WHERE Id LIKE 'QueryStoreDiagnosticsWatchdog%';"; + return Convert.ToInt32(await command.ExecuteScalarAsync(cancellationToken)); } private static async Task CreateProbeTableAsync(SqlConnection connection, string tableName, CancellationToken cancellationToken) From 9c566494236be97ae127fee3a1fbd825af4df0fd Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 25 Aug 2026 20:01:35 +0000 Subject: [PATCH 16/20] Rewrite ADR-2608 in plainer prose Applies the author's edits and reworks the wording to read less formally and to sit closer to the other ADRs in docs/arch. Halves the em-dash count, breaks the dense compound sentences in the decision into separate paragraphs, and drops rhetorical scaffolding. Restores the configuration-only decision to the Decision section. The adverse effects referred to keeping configuration out of dbo.Parameters while the decision itself no longer stated it, so a reader met the cost of a choice that had not been recorded. Context now says why Query Store data is a PHI concern specifically: plans capture compiled and runtime parameter values, so a plan can carry patient data even when no resource table was queried. That is the reason sanitization exists and is worth stating where the risk is introduced. Also records the lease include and exclude patterns as an adverse effect. They live in dbo.Parameters, belong to the shared lease rather than to this feature, and can leave the feature enabled and silent. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...608-query-store-performance-diagnostics.md | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/docs/arch/adr-2608-query-store-performance-diagnostics.md b/docs/arch/adr-2608-query-store-performance-diagnostics.md index c0283fdcec..52d88894db 100644 --- a/docs/arch/adr-2608-query-store-performance-diagnostics.md +++ b/docs/arch/adr-2608-query-store-performance-diagnostics.md @@ -8,46 +8,55 @@ Labels: [SQL](https://github.com/microsoft/fhir-server/labels/Area-SQL) ## Context -Diagnosing a slow FHIR service today requires Azure SQL Query Store data — the slowest recent queries, their execution plans, wait statistics, and statistics health. None of that reaches an on-call engineer or an automated SRE agent without a human who holds database credentials connecting to the customer's data plane and querying it by hand. That is slow during an incident and does not scale across a multi-tenant fleet. +Diagnosing a slow FHIR service today requires Azure SQL Query Store data: the slowest recent queries, their execution plans, wait statistics, and statistics health. Getting that data means connecting to the database, copying and pasting queries, and working by hand against a production system. That is slow and prone to errors. -The constraints that shaped the decision are as much operational as technical. The data plane holds PHI, and Query Store plan XML can embed literal parameter values, so anything that moves plans out of the database is a privacy boundary. The team deliberately operates the service without standing database access, so any design that requires a new identity with rights on customer databases is not merely an implementation detail — it is a permission model the organisation would have to review, provision per environment, rotate, and audit for as long as the feature exists. +Direct database access can also expose PHI or PII by accident. The risk is not only the resource data itself. Query Store captures compiled and runtime parameter values inside query plans, so a plan copied out of the database can carry patient data with it even when nobody queried a resource table. + +We look to simplify database performance analysis and reduce the possible PHI / PII exposure as part of this ADR. ## Options Considered -1. **External caller executing diagnostics stored procedures** — grant an outside identity, such as the SRE agent, a SQL role restricted to a fixed set of diagnostics stored procedures, and let it connect to the customer database directly. *(rejected: creates a new standing access path into the data plane)* -2. **Geneva Actions brokering the same stored procedures** — keep the procedures, but invoke them through Geneva Actions so the agent never holds SQL credentials itself. *(rejected: moves the credential but does not remove the access path)* -3. **In-process background job that emits diagnostics** — the server collects on its own schedule using the identity it already holds, sanitises plans in C#, and publishes the results through the existing metrics notification pipeline. *(chosen)* +1. **External caller executing diagnostics stored procedures** — grant an outside identity, such as the cloud SRE agent, a SQL role restricted to a fixed set of diagnostics stored procedures, and let it connect to the customer database directly. *(rejected: creates a new standing access path into the data plane)* +2. **PaaS Action layer brokering the same stored procedures** — keep the procedures, but invoke them through the existing PaaS Action broker so the agent never holds SQL credentials itself. *(rejected: moves the credential but does not remove the access path)* +3. **In-process background job that emits diagnostics** — the server collects on its own schedule using the identity it already holds, sanitizes plans in C#, and publishes the results through the existing metrics notification pipeline. *(chosen)* ## Decision We chose option 3: a watchdog-style background job inside the FHIR server, off by default, enabled and tuned entirely through configuration. -The deciding argument is that **it introduces no new access path**. The server is already authenticated to its own database and already runs scheduled background work there; diagnostics collection is additional work on an existing connection and an existing identity. Options 1 and 2 both require a principal that can reach the data plane from outside. Option 2 is genuinely better than option 1 — the agent never holds a credential — but Geneva Actions is a layer in front of the access path, not a replacement for it: the role, its grants, and its lifecycle still have to exist. A SQL reviewer on the team made the same point, that an outside force connecting in and running procedures is a permission model we do not otherwise operate. +The deciding argument is that it introduces no new access path. The server is already authenticated to its own database and already runs scheduled background work there, so diagnostics collection is more work on a connection and an identity that both already exist. Options 1 and 2 each need a principal that can reach the data plane from outside. Option 2 is better than option 1, because the agent never holds a credential itself, but the existing action path has no database access today. Choosing it would still mean opening a path that is not there now. + +Three things follow from that choice, and each of them reinforced it. + +Data leaves by push, through the notification pipeline that hosts already bind to, rather than by an inbound query into the data plane. The direction of trust stays the same as it is today. + +Plan sanitization moves from T-SQL into C#. There it is unit tested, it matches on element and attribute names rather than on a Showplan namespace that changes between SQL versions, and it fails closed by verifying its own output before anything is published. -Three consequences of that choice reinforced it. Data now leaves by **push through the existing notification pipeline** that hosts already bind to, rather than by an inbound query into the data plane, which keeps the direction of trust unchanged. **Plan sanitisation moves from T-SQL into C#**, where it is unit-testable, namespace-agnostic across SQL versions, and fails closed by verifying its own output before publishing. And enablement uses the **existing configuration surface**, so switching diagnostics on in an environment is an ordinary deployment change rather than a database operation. +Enablement uses the existing configuration surface, so turning diagnostics on in an environment is an ordinary deployment change rather than a database operation. -We further decided that **all settings live in configuration and none in `dbo.Parameters`**. The first iteration followed the existing watchdog convention of keeping runtime knobs in that table, which meant an operator had to run an `UPDATE` to arm the feature and meant the period was read from the database over the top of configuration. That reintroduces database writes for a feature whose purpose is to avoid needing database access, and splits the control surface in two. Honouring it required this watchdog to stop deriving from the shared `Watchdog` base class, whose initialisation privately writes its period rows and reads them back; the distributed lease that prevents duplicate collection across replicas was kept. +We also decided that every setting lives in configuration and none in `dbo.Parameters`. The first iteration followed the existing watchdog convention of keeping runtime values in that table. That meant an operator had to run an `UPDATE` to arm the feature, and the collection period was read back from the database over the top of the configured value. Both work against the goal: a feature whose purpose is to remove the need for database access should not require a write to the database to switch on, and configuration that is silently overridden by a stored row is not really configuration. Honoring this meant the watchdog could no longer derive from the shared `Watchdog` base class, whose initialization writes those rows and reads them back from a private, non-virtual step with no override hook. The distributed lease that stops every replica collecting the same data was kept. ## Consequences ### Benefits -- No new principal, role, firewall exception, or credential to provision, rotate, or audit; nothing outside the service gains data-plane access. +- No new principal, role, firewall exception, or credential to provision, rotate, or audit. Nothing outside the service gains data-plane access. - Diagnostics are enabled and tuned per environment through normal configuration, including an optional run window, and are off by default. - The PHI boundary is enforced in C#, is covered by unit tests, and fails closed rather than emitting an unverified plan. - Collection is single-instance through the existing lease, so notifications are not multiplied by replica count. ### Adverse effects -- Diagnostics cannot be pulled on demand. Data appears on the collection period — hourly by default — so an incident is served by data already being collected, not by an engineer asking a question and getting an immediate answer. A run window has to be configured in advance. +- Diagnostics cannot be pulled on demand. Data appears on the collection period, hourly by default, so an incident is served by data that was already being collected rather than by an engineer asking a question and getting an answer straight away. A run window has to be configured in advance. - Settings bind through `IOptions`, so changing them on a running host requires a restart. -- This watchdog no longer shares the `Watchdog` base class and therefore re-implements its timer and lease orchestration and will not inherit future improvements to it. That divergence is the cost of removing `dbo.Parameters`, and should be revisited if the base class itself moves to configuration. -- One shared type changed: `WatchdogLease` was constrained to `T : Watchdog` while using its type argument solely for `typeof(T).Name`. The constraint was relaxed so a self-scheduling component can still elect a single replica. It restricted nothing the class used, and every existing caller passes a `Watchdog` and is unaffected, but it is a shared-file change and reviewers should confirm they are comfortable with it. -- Collection depends on Query Store being enabled and in `READ_WRITE` state on the database; otherwise the job reports why it cannot collect and does nothing. +- This watchdog no longer shares the `Watchdog` base class, so it re-implements that class's timer and lease orchestration and will not pick up future improvements to it. That is the cost of keeping configuration out of `dbo.Parameters`, and it should be revisited if the base class itself moves to configuration. +- One shared type changed. `WatchdogLease` was constrained to `T : Watchdog` but used its type argument only for `typeof(T).Name`. The constraint was relaxed so that a component which schedules itself can still elect a single replica. It restricted nothing the class actually used, and every existing caller still satisfies it, but it is a change to a shared file and reviewers should confirm they are comfortable with it. +- Collection depends on Query Store being enabled and in `READ_WRITE` state on the database. Otherwise the job reports why it cannot collect and does nothing. +- One piece of pre-existing database state can still suppress collection silently. `dbo.AcquireWatchdogLease` honors watchdog lease include and exclude patterns held in `dbo.Parameters`. A worker excluded by such a row never becomes lease holder, so the feature can be enabled and stay quiet. This applies to every watchdog in the process and is not something this feature sets or reads, but it is the first thing to check on a long-lived database. ### Neutral effects -- The emitted notifications are contracts that hosts bind to; this repository prescribes no sink. +- The emitted notifications are contracts that hosts bind to. This repository prescribes no sink. - The lease continues to write to its own table. That is runtime coordination rather than configuration, and is not part of what this decision removed. ## References From e5c9cbd4cb8a22829243e497ecefaf22a9073d22 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 25 Aug 2026 20:10:43 +0000 Subject: [PATCH 17/20] Document why statistics health reporting is capped StatisticsHealthCount looked like an arbitrary truncation of a set that could be much larger. Records what it actually is and what it costs. Each reported row is published as its own notification, so the setting multiplies emission volume per collection, per database, per host. The schema defines roughly a hundred index-backed statistics before SQL Server adds auto-created column statistics, so reporting everything would make one collection several hundred notifications. ADR-2605 records the consequence of that pattern: a shared metric account was throttled and monitoring degraded for both FHIR and DICOM. Also records a bias in the ordering. Ranking is by modification ratio, so a small heavily-churned table outranks a large one that has drifted less proportionally, and small busy tables can fill the report while a consequential stale statistic on a large table falls below the cut. Noted rather than changed, since which definition of "worst" is right depends on what is being chased. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 3d6b528c13..7ec08a8c2b 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -79,7 +79,7 @@ The only database row this feature causes to exist is its **lease**, in `dbo.Wat | `MinDurationMilliseconds` | `1000` | Minimum weighted average duration for a plan to be reported. A negative value is treated as `0` — which makes every query qualify — and warns. | | `IncludeQueryPlans` | `true` | Whether sanitized Showplan XML is emitted. | | `IncludeStatisticsHealth` | `true` | Whether statistics metadata is emitted. | -| `StatisticsHealthCount` | `20` | Number of statistics rows to report per tick. | +| `StatisticsHealthCount` | `20` | Worst-ranked statistics rows to report per tick. Each row is one notification, so this directly multiplies emission volume; see [Why the count is capped](#why-the-count-is-capped). | | `RunStartDate` | `null` (unset) | Inclusive start of the optional run window. Ticks before it collect nothing. `null` means no lower bound. | | `RunEndDate` | `null` (unset) | **Exclusive** end of the optional run window. Ticks at or after it collect nothing. `null` means no upper bound. | @@ -188,6 +188,16 @@ Statistics are read from `sys.stats` with an `OUTER APPLY` to `sys.dm_db_stats_p Modification percentage is left null when the row count is null or zero rather than being reported as zero, so "no data" is distinguishable from "not stale". Percentages above 100 are preserved; they are a legitimate signal that a table has churned more than its cardinality. +#### Why the count is capped + +`StatisticsHealthCount` is a cap on what is *reported*, not on what is examined: the ordering runs across every qualifying statistic and only the worst are emitted. The cap matters because each reported row is published as its own `StatisticsHealthNotification`, so the setting is a direct multiplier on emission volume — per collection, per database, per host. + +The schema alone defines roughly a hundred index-backed statistics across its user tables, and SQL Server adds auto-created column statistics on top of that as queries run, so the full set on a busy database is comfortably several hundred. Reporting all of them would turn one collection into several hundred notifications, and a fleet multiplies that by database count. `docs/arch/adr-2605-metric-emission-rate-limiting.md` records what that costs: a high-volume emission pattern throttled a *shared* metric account and degraded monitoring for both the FHIR and DICOM services. Metric events are charged on receipt, so volume is both a cost and an availability concern. + +Capping is therefore the right shape, and the ordering is what makes a small cap usable — the reported rows are the worst offenders rather than an arbitrary slice. Statistics with no readable row count sort last, so empty and unsampled tables do not consume the budget. + +One bias is worth knowing when reading the output. Ranking is by *ratio*, so a small table that churns heavily outranks a large one that has drifted less proportionally: ten rows with a hundred modifications scores 10.0, while a hundred-million-row table with twenty million modifications scores 0.2, even though the second is far more likely to distort a plan. A handful of small, busy tables can therefore fill the report while a consequential stale statistic on a large table sits below the cut. Raising `StatisticsHealthCount` widens the window, at the emission cost described above. If large-table staleness is what is being chased, the ordering — not the cap — is the thing to revisit. + ### Failure containment `WatchdogsBackgroundService` cancels **every** watchdog if any one of them fails. A diagnostic feature must never be able to take down transaction or cleanup watchdogs, so the collection body contains its own failures: missing views and permission denials are logged and the tick returns rather than propagating. From 53efdcbe32370598aab9f39e6ee1045376e37674 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 25 Aug 2026 20:50:46 +0000 Subject: [PATCH 18/20] Emit query store diagnostics as structured logs instead of metrics The payload was never metric shaped. QueryText is unbounded high-cardinality text, SanitizedQueryPlan is an XML document, and TopWaitCategory is a high-cardinality string. None of those work as metric dimensions, and metric events are charged on receipt. adr-2605-metric-emission-rate-limiting.md records what volume on that pipeline costs: a shared metric account was throttled and monitoring degraded for both FHIR and DICOM. The three payload types no longer implement IMetricsNotification, and the watchdog no longer takes IMediator at all. Since the types are no longer a cross-assembly contract they move out of Core into the watchdog folder as internal types and lose the FhirOperation and ResourceType members that existed only to satisfy the metrics interface. Leaving public INotification types in Core that nothing publishes would have been misleading. They are new in this PR and have never shipped, so nothing depended on them. Slow queries and plans are emitted one structured line each, with named properties so each field stays queryable as a column. Statistics rows are batched into a JSON array, StatisticsHealthBatchSize rows per line, each line carrying its page number, page count and total row count so a partial final page is distinguishable from a set cut short. Batching suits these rows because they are small, uniform and free of free text; plan XML could not make that promise. Batch size is clamped to 64. Batching trades record count for record size, and an unbounded batch would rebuild the single oversized record that is the reason plan XML is not batched at all. A serialized row is a little under 400 bytes, so 64 keeps a page inside the 32 KB budget already applied to other large fields. Clamping pages the rows rather than dropping them. Sanitization and truncation are unchanged. The same bytes are emitted; only the destination differs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 109 ++++-- ...608-query-store-performance-diagnostics.md | 25 +- .../QueryStoreDiagnosticsConfiguration.cs | 9 + .../appsettings.json | 3 +- .../QueryStoreDiagnosticsPeriodTests.cs | 2 - .../QueryStoreDiagnosticsRunWindowTests.cs | 2 - ...reDiagnosticsStatisticsHealthBatchTests.cs | 271 +++++++++++++++ ...ueryStoreDiagnosticsWaitStatisticsTests.cs | 102 ++++-- .../Watchdogs/QueryPlanDiagnostics.cs} | 22 +- .../QueryStoreDiagnosticsWatchdog.cs | 201 ++++++++--- .../Watchdogs/SlowQueryDiagnostics.cs} | 23 +- .../Watchdogs/StatisticsHealthDiagnostics.cs} | 23 +- .../QueryStoreDiagnosticsWatchdogTests.cs | 311 ++++++++++++++---- 13 files changed, 895 insertions(+), 208 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs rename src/{Microsoft.Health.Fhir.Core/Features/Metrics/QueryPlanNotification.cs => Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanDiagnostics.cs} (76%) rename src/{Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs => Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SlowQueryDiagnostics.cs} (84%) rename src/{Microsoft.Health.Fhir.Core/Features/Metrics/StatisticsHealthNotification.cs => Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/StatisticsHealthDiagnostics.cs} (74%) diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 7ec08a8c2b..4d1c9a3544 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -2,7 +2,7 @@ ## Status -Agreed baseline for implementation. This document defines an opt-in background worker that emits Azure SQL Query Store diagnostics from inside the FHIR server, the enablement model, the emitted contracts, the disclosure boundary, and the repository ownership split. +Agreed baseline for implementation. This document defines an opt-in background worker that emits Azure SQL Query Store diagnostics from inside the FHIR server, the enablement model, the emitted log records, the disclosure boundary, and the repository ownership split. This specification **supersedes an earlier pull-based design** in which three stored procedures were called by an external caller. See [Rejected alternative](#rejected-alternative-caller-invoked-stored-procedures). @@ -26,7 +26,7 @@ Related internal guidance: ## Design -A **watchdog** — the repository's existing leased background-worker pattern — periodically reads Query Store and statistics metadata and **pushes** the results out as metrics notifications. It runs inside the FHIR server process on the server's **existing** SQL identity. +A **watchdog** — the repository's existing leased background-worker pattern — periodically reads Query Store and statistics metadata and writes the results out as **structured log records**. It runs inside the FHIR server process on the server's **existing** SQL identity. ```text FHIR server instance (lease holder) @@ -37,8 +37,8 @@ FHIR server instance (lease holder) ├── sys.stats / sys.dm_db_stats_properties └── QueryPlanSanitizer (C#) strips parameter values │ - └── IMediator.PublishAsync(IMetricsNotification) - └── host-supplied handler (PaaS -> Geneva / Log Analytics) + └── ILogger structured log records + └── the host's existing log pipeline (PaaS -> Geneva / Log Analytics) ``` The critical property is that **nothing new connects inbound to the database**. The work is done by the service that already holds a connection, so the feature introduces no new authentication path, no new database principal, and no new grant. @@ -47,9 +47,9 @@ The critical property is that **nothing new connects inbound to the database**. `WatchdogsBackgroundService` and `WatchdogLease` already provide what this feature needs, and each of these would otherwise have to be reinvented: -- **Single-runner election.** `WatchdogLease` ensures exactly one instance in a multi-instance deployment performs the work, so an eight-instance service does not issue eight concurrent Query Store scans and emit eight copies of every notification. The lease is invisible runtime coordination rather than configuration: it holds nothing an operator sets, and it lives in `dbo.WatchdogLeases` behind `dbo.AcquireWatchdogLease`, shared with every other watchdog. +- **Single-runner election.** `WatchdogLease` ensures exactly one instance in a multi-instance deployment performs the work, so an eight-instance service does not issue eight concurrent Query Store scans and emit eight copies of every diagnostics line. The lease is invisible runtime coordination rather than configuration: it holds nothing an operator sets, and it lives in `dbo.WatchdogLeases` behind `dbo.AcquireWatchdogLease`, shared with every other watchdog. - **A managed background timer.** `FhirTimer` supplies the randomized start-up stagger that keeps replicas from collecting on the same second, and catches whatever a tick throws so that a failed collection costs one tick rather than the process. -- **A precedent for emitting SQL telemetry.** `GeoReplicationLagWatchdog` reads a SQL view on a timer and publishes an `IMetricsNotification`; the host binds a handler that forwards it. This feature is the same shape. +- **A precedent for emitting SQL telemetry.** `GeoReplicationLagWatchdog` reads a SQL view on a timer and emits what it finds. This feature has the same shape, but emits **logs rather than metric notifications**; see [Why logs rather than metrics](#why-logs-rather-than-metrics). A single lag figure is metric-shaped, and query text and plan XML are not. **It deliberately does not derive from `Watchdog`.** That base class inserts `{Name}.PeriodSec` and `{Name}.LeasePeriodSec` into `dbo.Parameters` on every start and then reads the period back **over** the configured value, from a private, non-virtual initialization step with no override hook. This feature is configured exclusively from configuration and writes nothing to the database, so it owns a `FhirTimer` and a `WatchdogLease` directly and reproduces the rest of what the base class did — the lease-holder gate, the capped randomized stagger, and the per-tick timing line — in its own `ExecuteAsync` and tick handler. `WatchdogLease` uses its type argument only to derive the lease resource name from `typeof(T).Name`; its former `T : Watchdog` constraint restricted nothing it actually used, so it was relaxed to admit a self-scheduling component. Every other caller passes a `Watchdog` and is unaffected, and the timer and base class are used unmodified. @@ -79,7 +79,8 @@ The only database row this feature causes to exist is its **lease**, in `dbo.Wat | `MinDurationMilliseconds` | `1000` | Minimum weighted average duration for a plan to be reported. A negative value is treated as `0` — which makes every query qualify — and warns. | | `IncludeQueryPlans` | `true` | Whether sanitized Showplan XML is emitted. | | `IncludeStatisticsHealth` | `true` | Whether statistics metadata is emitted. | -| `StatisticsHealthCount` | `20` | Worst-ranked statistics rows to report per tick. Each row is one notification, so this directly multiplies emission volume; see [Why the count is capped](#why-the-count-is-capped). | +| `StatisticsHealthCount` | `20` | Worst-ranked statistics rows **collected and reported** per tick; see [Why the count is capped](#why-the-count-is-capped). | +| `StatisticsHealthBatchSize` | `20` | How many of those rows are packed into each log line. This is **not** a second cap on what is collected: rows beyond the batch size are emitted on further lines, paginated. A non-positive value is reported with a warning and the `20` default is used, because a batch size cannot pack a row. Values above `64` are clamped, also with a warning; see [Batch size is capped](#batch-size-is-capped). | | `RunStartDate` | `null` (unset) | Inclusive start of the optional run window. Ticks before it collect nothing. `null` means no lower bound. | | `RunEndDate` | `null` (unset) | **Exclusive** end of the optional run window. Ticks at or after it collect nothing. `null` means no upper bound. | @@ -122,27 +123,61 @@ The window state is logged **only when it changes** — not open yet, open, clos **The window boundaries take effect without a restart, but the configured values do not.** These settings bind through `IOptions`, so the *values* are read once at process start and editing configuration afterwards has no effect until the host restarts — the same as every other setting in this feature. What changes without a restart is the *clock*: a window configured before the host started will open and close on schedule while the process keeps running, because each tick re-evaluates the fixed boundaries against the current time. Changing a boundary on a running host still requires a restart. -## Emitted contracts +## Emitted log records -Three notification types implement `IMetricsNotification`, each reporting `FhirOperation` `query-store-diagnostics` and `ResourceType` `System`. Hosts bind handlers to route them; the OSS repository does not prescribe a sink. +Everything this feature produces is a **structured log record**, written through the `ILogger` the host already configures. Nothing is emitted as a metric event, and there is no handler to bind: whatever a deployment already does with FHIR server logs, it does with these. -### `SlowQueryNotification` +All three payload lines are emitted at **information** level. The warnings this feature raises — misconfiguration, an unavailable Query Store, a failed wait read, a plan that would not sanitize — remain at **warning** level and are unaffected by how the payload is emitted. -One per slow plan per tick. Carries `QueryId`, `PlanId`, execution count, total/average/maximum duration, total/average CPU, total/average logical reads, total/average wait time, top wait category, `WaitStatisticsStatus`, the Query Store query text, and the collection window bounds. +The payload shapes are `SlowQueryDiagnostics`, `QueryPlanDiagnostics`, and `StatisticsHealthDiagnostics`, in `Microsoft.Health.Fhir.SqlServer/Features/Watchdogs`. They are `internal` and live beside the watchdog because they are log payload shapes, not contracts another assembly binds to. + +### Why logs rather than metrics + +- **Cost and blast radius.** Metric events are charged on receipt. `docs/arch/adr-2605-metric-emission-rate-limiting.md` records a high-volume emission pattern throttling a *shared* metric account and degrading monitoring for both the FHIR and DICOM services, so volume on that pipeline is an availability concern as well as a bill. Logs are the cheap place to start, and moving a data point to a metric later is easy in a way that recovering a throttled account is not. +- **The payload is not metric-shaped.** `QueryText` is unbounded free text, `SanitizedQueryPlan` is an XML document, and `TopWaitCategory` is a high-cardinality string. Those are log fields. Carrying them as metric dimensions is a cardinality incident waiting to happen. +- **Nothing here is a rate.** These are periodic snapshots meant to be read and correlated by a responder during an investigation, not aggregated into a time series. + +### Slow query + +One line per slow plan per tick, beginning `QueryStoreDiagnosticsWatchdog slow query.`. + +Every field is its own **named** log property, so each lands as its own queryable column in Kusto or Log Analytics rather than inside a serialized blob: `QueryId`, `PlanId`, `ExecutionCount`, `TotalDurationMilliseconds`, `AverageDurationMilliseconds`, `MaxDurationMilliseconds`, `TotalCpuMilliseconds`, `AverageCpuMilliseconds`, `TotalLogicalReads`, `AverageLogicalReads`, `TotalWaitMilliseconds`, `AverageWaitMilliseconds`, `TopWaitCategory`, `WaitStatisticsStatus`, `QueryTextTruncated`, `QueryTextLength`, `IntervalStart`, `IntervalEnd`, `DiagnosticsTimestamp`, and `QueryText`. + +These lines are **not** batched. At the default `SlowQueryCount` of 10 a line per row is cheap, and column-level queryability is the whole reason to emit structured logs rather than JSON documents. `QueryText` is placed last so that everything an operator scans for is readable ahead of the one unbounded field on the line. `WaitStatisticsStatus` describes why the wait fields look the way they do: `Available` when wait statistics were read for the plan, `Unavailable` when the wait query succeeded but returned no row for it (wait capture off, or no waits accrued), and `Failed` when the wait query itself threw. Without it, "wait capture is switched off" and "the wait query has been failing for a month" are indistinguishable, because both leave the wait fields null. `QueryId` and `PlanId` are the join keys back into Query Store, and into the `QueryStoreRuntimeStatistics` stream already exported to Log Analytics, so a responder can correlate an emitted slow query with the existing diagnostic-settings telemetry. -### `QueryPlanNotification` +### Query plan + +One line per reported plan per tick when `IncludeQueryPlans` is set, beginning `QueryStoreDiagnosticsWatchdog query plan.`. Named properties: `QueryId`, `PlanId`, `SanitizationStatus`, `QueryPlanTruncated`, `OriginalQueryPlanLength`, `SanitizedQueryPlanLength`, `DiagnosticsTimestamp`, and `SanitizedQueryPlan`. + +These lines are **not** batched either, and for a different reason from the slow-query lines: the sanitized XML is capped at the field length rather than being small, so packing several plans into one record would risk producing a single oversized log record that the pipeline drops or truncates as a whole. -One per reported plan per tick when `IncludeQueryPlans` is set. Carries `QueryId`, `PlanId`, the sanitized Showplan XML, a truncation flag, the raw and sanitized plan lengths, and a sanitization status. +A line is emitted for unsuccessful sanitization as well, with null XML and a status of `PlanXmlUnavailable`, `InvalidXml`, or `VerificationFailed`, so a sanitization failure is observable rather than silent. That failure is *also* logged as its own warning, so systematic sanitizer breakage does not look like "plans are simply unavailable" to someone who is not reading `SanitizationStatus`. + +### Statistics health -Notifications are emitted for unsuccessful sanitization as well, with null XML and a status of `PlanXmlUnavailable`, `InvalidXml`, or `VerificationFailed`, so a sanitization failure is observable rather than silent. +Emitted in **batches**, beginning `QueryStoreDiagnosticsWatchdog statistics health.`. Each line carries `StatisticsHealthBatchSize` rows serialized as a compact JSON array in a single log property, `StatisticsHealthRows`, alongside the pagination properties described below. -### `StatisticsHealthNotification` +These rows are batched where the other two payloads are not because they are small, uniform, and contain no free text, so a batch of them has a predictable size. The cost of batching is that the rows arrive as a serialized blob rather than as queryable columns, which is affordable precisely because the fields are few, uniform, and cheap to re-parse; it would not be affordable for query text or plan XML. -One per statistics object per tick when `IncludeStatisticsHealth` is set. Carries schema, table, and statistics name, last-updated timestamp, rows, rows sampled, modification counter, modification percentage, and the auto-created / user-created / from-index / filtered flags. +Each row carries schema, table, and statistics name, last-updated timestamp, rows, rows sampled, modification counter, modification percentage, the auto-created / user-created / from-index / filtered flags, and the collection timestamp. The timestamp is on the row rather than only on the line so that a row stays self-describing once it is lifted out of the batch it arrived in. + +When more rows are collected than fit in one batch, several lines are emitted and each one says where it sits in the set: + +| Property | Meaning | +| --- | --- | +| `StatisticsHealthPage` | The **1-based** page number of this line. | +| `StatisticsHealthPageCount` | How many lines the collection was emitted across. | +| `StatisticsHealthPageRowCount` | How many rows this line carries. | +| `StatisticsHealthRowCount` | How many rows were collected in total, across every page. | +| `StatisticsHealthRows` | The rows themselves, as a compact JSON array. | + +Carrying the totals on **every** line is what lets a reader distinguish a legitimately short final page from a set that was cut short by a host that died mid-collection: 18 rows on page 3 of 3 of a 58-row collection is complete, while pages 1 and 2 of 3 arriving alone is not. No line is emitted when nothing was collected — the per-tick summary already reports a count of zero, and an empty page would only make the pages harder to count. + +`DiagnosticsTimestamp` on the slow-query and query-plan lines, and `Timestamp` inside each statistics row, are the moment the watchdog collected the data. They are deliberately not named `Timestamp` at the log-property level, because that name collides with the ingestion timestamp log pipelines supply for every record. ### Field size @@ -178,9 +213,9 @@ Wait statistics are collected by a **separate, best-effort** query and merged in `Failed` is reserved for that case: a wait read that actually broke. The ordinary outcomes of wait capture being turned off, or of no waits having accrued for a plan inside the window, are an empty result set rather than a failure — those plans carry `WaitStatisticsStatus` of `Unavailable`, with null wait fields and no warning logged. -`SqlException` is caught broadly there on purpose, so that a transient wait-query failure cannot abort the tick and suppress the runtime metrics. The trade-off accepted is that timeouts, deadlocks, permission denials and missing views are not distinguished from one another at that point; the warning carries the exception, and the notification carries the status. +`SqlException` is caught broadly there on purpose, so that a transient wait-query failure cannot abort the tick and suppress the runtime statistics. The trade-off accepted is that timeouts, deadlocks, permission denials and missing views are not distinguished from one another at that point; the warning carries the exception, and the slow-query line carries the status. -Wait capture is retained locally, rather than deferred entirely to the `QueryStoreWaitStatistics` Log Analytics stream, so that a single emitted slow-query record is self-contained: runtime metrics, waits, query text, and plan identity arrive together without a join against a second telemetry source. +Wait capture is retained locally, rather than deferred entirely to the `QueryStoreWaitStatistics` Log Analytics stream, so that a single emitted slow-query line is self-contained: runtime statistics, waits, query text, and plan identity arrive together without a join against a second telemetry source. ### Statistics health @@ -190,19 +225,29 @@ Modification percentage is left null when the row count is null or zero rather t #### Why the count is capped -`StatisticsHealthCount` is a cap on what is *reported*, not on what is examined: the ordering runs across every qualifying statistic and only the worst are emitted. The cap matters because each reported row is published as its own `StatisticsHealthNotification`, so the setting is a direct multiplier on emission volume — per collection, per database, per host. +`StatisticsHealthCount` is a cap on what is *reported*, not on what is examined: the ordering runs across every qualifying statistic and only the worst are emitted. It is also the only cap on the number of rows. `StatisticsHealthBatchSize` decides how many rows share a log line, not how many rows exist, so raising it never reports more and lowering it never reports less — the two settings are easy to confuse and do quite different things. + +The reason to cap is weaker than it was when each row was emitted as its own metric event, and it is worth being precise about what changed. Rows are now packed into batched log lines, so a row no longer costs a charged metric event and the per-row emission argument that `docs/arch/adr-2605-metric-emission-rate-limiting.md` supports no longer applies here in that form. What batching removes is **record count**, not bytes: 300 rows at a batch size of 20 is 15 log records instead of 300, but it is still 300 rows of ingested and retained data. + +That residual volume is what the cap is for now. The schema alone defines roughly a hundred index-backed statistics across its user tables, and SQL Server adds auto-created column statistics on top of that as queries run, so the full set on a busy database is comfortably several hundred — every collection, on every database, on every host that holds the lease, for as long as the feature is enabled. Log ingestion and retention are charged by volume, and a fleet multiplies the figure by database count, so reporting everything every hour is a real ongoing cost even though it is a much smaller one than it would have been on the metrics pipeline. The second cost is readability: several hundred rows an hour of mostly healthy statistics is a stream nobody reads, which defeats the purpose of collecting them. + +Capping is therefore still the right shape, and the ordering is what makes a small cap usable — the reported rows are the worst offenders rather than an arbitrary slice. Statistics with no readable row count sort last, so empty and unsampled tables do not consume the budget. + +One bias is worth knowing when reading the output. Ranking is by *ratio*, so a small table that churns heavily outranks a large one that has drifted less proportionally: ten rows with a hundred modifications scores 10.0, while a hundred-million-row table with twenty million modifications scores 0.2, even though the second is far more likely to distort a plan. A handful of small, busy tables can therefore fill the report while a consequential stale statistic on a large table sits below the cut. Raising `StatisticsHealthCount` widens the window, at the volume cost described above; on logs that is a defensible thing to do for the length of an investigation, which it was not when every extra row was a metric event. If large-table staleness is what is being chased, the ordering — not the cap — is the thing to revisit. + +#### Batch size is capped -The schema alone defines roughly a hundred index-backed statistics across its user tables, and SQL Server adds auto-created column statistics on top of that as queries run, so the full set on a busy database is comfortably several hundred. Reporting all of them would turn one collection into several hundred notifications, and a fleet multiplies that by database count. `docs/arch/adr-2605-metric-emission-rate-limiting.md` records what that costs: a high-volume emission pattern throttled a *shared* metric account and degraded monitoring for both the FHIR and DICOM services. Metric events are charged on receipt, so volume is both a cost and an availability concern. +`StatisticsHealthBatchSize` is clamped to 64 rows per line, with a warning naming the configured value when the clamp bites. -Capping is therefore the right shape, and the ordering is what makes a small cap usable — the reported rows are the worst offenders rather than an arbitrary slice. Statistics with no readable row count sort last, so empty and unsampled tables do not consume the budget. +The reason is the same one that keeps plan XML out of a batch. Batching trades record count for record size, and a large enough batch recreates exactly the oversized record that batching plans was rejected for: a single line that a sink may truncate or reject, taking every row on it with it. A typical serialized statistics row is a little under 400 bytes, so 64 rows keeps a full page well inside the 32 KB budget the feature already applies to its other large fields. -One bias is worth knowing when reading the output. Ranking is by *ratio*, so a small table that churns heavily outranks a large one that has drifted less proportionally: ten rows with a hundred modifications scores 10.0, while a hundred-million-row table with twenty million modifications scores 0.2, even though the second is far more likely to distort a plan. A handful of small, busy tables can therefore fill the report while a consequential stale statistic on a large table sits below the cut. Raising `StatisticsHealthCount` widens the window, at the emission cost described above. If large-table staleness is what is being chased, the ordering — not the cap — is the thing to revisit. +Clamping never drops rows. A batch size above the cap simply produces more pages, and the pagination properties still account for every row. Extra lines are the cheap thing here, which is the whole reason for preferring logs in the first place. ### Failure containment `WatchdogsBackgroundService` cancels **every** watchdog if any one of them fails. A diagnostic feature must never be able to take down transaction or cleanup watchdogs, so the collection body contains its own failures: missing views and permission denials are logged and the tick returns rather than propagating. -The missing-view handler spans the whole collection rather than each individual read, so the aborting read can be the last one, after slow queries and plans have already been published. Its warning is worded to hold in that case too: it reports that collection was aborted and that whatever had already been emitted was still published, rather than claiming that nothing was collected. +The missing-view handler spans the whole collection rather than each individual read, so the aborting read can be the last one, after slow queries and plans have already been emitted. Its warning is worded to hold in that case too: it reports that collection was aborted and that whatever had already been emitted was still logged, rather than claiming that nothing was collected. That containment covers **per-tick collection**, which is where all of this feature's own database work happens: `FhirTimer` catches whatever a tick throws and keeps ticking, so a failed collection costs one tick. The lease renewal is the only other database call, and it runs on the lease's own `FhirTimer` with the same per-tick catch. There is no initialization step left to fail outside either catch. Watchdogs that derive from `Watchdog` do have one — `ExecuteAsync` awaits `InitParamsAsync`, which seeds `dbo.Parameters`, *before* and *outside* the per-tick catch, so a throw there faults the watchdog task and `WatchdogsBackgroundService` cancels the rest — and not deriving from it removes that failure mode here along with the writes. @@ -220,7 +265,7 @@ The cost is one collection per period against the primary — hourly by default A non-positive `SlowQueryCount` or `StatisticsHealthCount` disables the corresponding section: the round-trip is skipped entirely rather than issued as a `TOP (0)` query whose empty result would be indistinguishable from a healthy one. Both cases are logged as a warning once per tick, and a section turned off deliberately through `IncludeQueryPlans` or `IncludeStatisticsHealth` is logged at information level. Misconfiguration is never fatal: a diagnostics feature must not fail the host. -A completed tick logs one information-level line carrying the collection window and the counts of slow queries, plans and statistics rows published, **including zeros**, so that "the watchdog has been dead for three days" is distinguishable from "there were no slow queries". The plan count is the number of plans that actually carried sanitized XML, not the number of plan notifications published, so it is deliberately lower than the slow-query count whenever Query Store held no plan for a query or sanitization rejected one. +A completed tick logs one information-level line carrying the collection window and the counts of slow queries, plans and statistics rows emitted, **including zeros**, so that "the watchdog has been dead for three days" is distinguishable from "there were no slow queries". The plan count is the number of plans that actually carried sanitized XML, not the number of plan lines emitted, so it is deliberately lower than the slow-query count whenever Query Store held no plan for a query or sanitization rejected one. The statistics count is the number of **rows** collected, not the number of batch lines they were emitted across. ## Sanitization @@ -250,7 +295,7 @@ Statistics histogram values are never read, because `range_high_key` contains ac The watchdog runs on the FHIR server's existing SQL connection and requires no additional database principal, role, or grant. Reading the Query Store catalog views requires `VIEW DATABASE STATE`, which the service identity already holds; where it does not, the permission denial is contained and logged rather than being fatal. -Because this is an outbound push from a process that is already trusted with the data, the previous design's audit requirements do not apply. There is no external caller to attribute, and the emitted notifications are themselves the operational record. +Because this is an outbound emission from a process that is already trusted with the data, the previous design's audit requirements do not apply. There is no external caller to attribute, and the emitted log records are themselves the operational record. ## Repository ownership @@ -258,7 +303,7 @@ Because this is an outbound push from a process that is already trusted with the - the watchdog, its inline SQL, and the C# sanitizer; - the configuration class and its defaults; -- the three notification contracts; +- the three log payload shapes and the structured lines they are emitted on; - unit tests for sanitization and integration tests against a live SQL Server; and - this specification. @@ -266,7 +311,7 @@ Nothing here is PaaS-specific, and no PaaS identity, storage account, or rollout ### `fhir-paas` -- binding notification handlers and routing the emissions to Geneva or Log Analytics; +- routing the FHIR server log stream to Geneva or Log Analytics, and any parsing of the emitted lines built on top of it; - setting the configuration gate per environment and ring; - setting the collection settings, including the run window, for an investigation; - retention, access control, and downstream handling of emitted query text and plans; and @@ -275,7 +320,7 @@ Nothing here is PaaS-specific, and no PaaS identity, storage account, or rollout ### Rollout 1. Merge the OSS change. The feature ships disabled. -2. Bind a handler and configure routing in `fhir-paas`. +2. Confirm the FHIR server log stream is routed where the investigation needs it. There is no handler to bind. 3. Enable the configuration gate in a test ring and confirm emission volume and field sizes. 4. Enable it for the deployment under investigation through configuration — bounding it with `RunStartDate` and `RunEndDate` when the investigation is time-boxed — and restart that deployment for the change to take effect. @@ -303,7 +348,7 @@ Secondary benefits of the change: - plan sanitization moves from T-SQL to C#, where it is more reliable and far easier to test; - the persistent SQL surface, the database role, and the schema version all disappear; and -- results are pushed into telemetry continuously rather than requiring someone to be connected and asking at the moment the problem is happening. +- results are emitted into telemetry continuously rather than requiring someone to be connected and asking at the moment the problem is happening. ## Testing requirements @@ -319,9 +364,9 @@ Secondary benefits of the change: ### Collection, integration tested against live SQL -1. With Query Store enabled and a deliberately slow query executed, a `SlowQueryNotification` is emitted carrying a matching `QueryId`/`PlanId`. -2. A `QueryPlanNotification` is emitted for that plan with status `Sanitized`. -3. `StatisticsHealthNotification` rows are emitted for user tables. +1. With Query Store enabled and a deliberately slow query executed, a slow-query line is emitted carrying a matching `QueryId`/`PlanId`. +2. A query-plan line is emitted for that plan with status `Sanitized`. +3. Statistics-health rows are emitted for user tables, batched and paginated, with page numbers and totals that account for every collected row. 4. The watchdog performs no work when the configuration gate is off, and starting it creates no `dbo.Parameters` row and reads none. 5. A non-`READ_WRITE` Query Store state is handled without error and without emission. 6. Wait-statistic unavailability degrades to null wait fields while runtime results are still emitted, and `WaitStatisticsStatus` reports which of the three outcomes occurred. diff --git a/docs/arch/adr-2608-query-store-performance-diagnostics.md b/docs/arch/adr-2608-query-store-performance-diagnostics.md index 52d88894db..f9e4cf58f0 100644 --- a/docs/arch/adr-2608-query-store-performance-diagnostics.md +++ b/docs/arch/adr-2608-query-store-performance-diagnostics.md @@ -18,7 +18,7 @@ We look to simplify database performance analysis and reduce the possible PHI / 1. **External caller executing diagnostics stored procedures** — grant an outside identity, such as the cloud SRE agent, a SQL role restricted to a fixed set of diagnostics stored procedures, and let it connect to the customer database directly. *(rejected: creates a new standing access path into the data plane)* 2. **PaaS Action layer brokering the same stored procedures** — keep the procedures, but invoke them through the existing PaaS Action broker so the agent never holds SQL credentials itself. *(rejected: moves the credential but does not remove the access path)* -3. **In-process background job that emits diagnostics** — the server collects on its own schedule using the identity it already holds, sanitizes plans in C#, and publishes the results through the existing metrics notification pipeline. *(chosen)* +3. **In-process background job that emits diagnostics** — the server collects on its own schedule using the identity it already holds, sanitizes plans in C#, and writes the results out as structured logs. *(chosen)* ## Decision @@ -28,12 +28,22 @@ The deciding argument is that it introduces no new access path. The server is al Three things follow from that choice, and each of them reinforced it. -Data leaves by push, through the notification pipeline that hosts already bind to, rather than by an inbound query into the data plane. The direction of trust stays the same as it is today. +Data leaves by push, as structured log records on the logger the server already writes to, rather than by an inbound query into the data plane. The direction of trust stays the same as it is today. -Plan sanitization moves from T-SQL into C#. There it is unit tested, it matches on element and attribute names rather than on a Showplan namespace that changes between SQL versions, and it fails closed by verifying its own output before anything is published. +Plan sanitization moves from T-SQL into C#. There it is unit tested, it matches on element and attribute names rather than on a Showplan namespace that changes between SQL versions, and it fails closed by verifying its own output before anything is emitted. Enablement uses the existing configuration surface, so turning diagnostics on in an environment is an ordinary deployment change rather than a database operation. +We also decided to emit the diagnostics as logs rather than as metric events. Three reasons, in order of weight. + +Metric events are charged when they are received, and the volume here is not small. `docs/arch/adr-2605-metric-emission-rate-limiting.md` records a high volume emission pattern that throttled a shared metric account and degraded monitoring for both the FHIR and DICOM services. That is an availability problem as well as a bill, and it is the kind of problem that is easier to avoid than to recover from. + +The data was never metric shaped. Query text is unbounded free text, a sanitized plan is an XML document, and the top wait category is a high cardinality string. Those belong in log fields. Putting them in metric dimensions invites a cardinality incident. Nothing collected here is a rate either. These are periodic snapshots that a responder reads during an investigation. + +Logs are also the cheap place to start. If a specific number later turns out to be worth alerting on, promoting it to a metric is a small change. Recovering a throttled metric account is not. + +The statistics health rows are the one payload that is batched. They are small, uniform, and free of free text, so several of them fit on one line as a JSON array without any risk of an oversized record. Each line carries its page number, the page count, and the total row count, so a reader can tell a short final page from a set that was cut short. The slow query and plan lines are not batched. Each field there is its own named log property, which is what keeps it queryable as a column, and plan XML is large enough that batching it would risk a single oversized record. + We also decided that every setting lives in configuration and none in `dbo.Parameters`. The first iteration followed the existing watchdog convention of keeping runtime values in that table. That meant an operator had to run an `UPDATE` to arm the feature, and the collection period was read back from the database over the top of the configured value. Both work against the goal: a feature whose purpose is to remove the need for database access should not require a write to the database to switch on, and configuration that is silently overridden by a stored row is not really configuration. Honoring this meant the watchdog could no longer derive from the shared `Watchdog` base class, whose initialization writes those rows and reads them back from a private, non-virtual step with no override hook. The distributed lease that stops every replica collecting the same data was kept. ## Consequences @@ -43,12 +53,14 @@ We also decided that every setting lives in configuration and none in `dbo.Param - No new principal, role, firewall exception, or credential to provision, rotate, or audit. Nothing outside the service gains data-plane access. - Diagnostics are enabled and tuned per environment through normal configuration, including an optional run window, and are off by default. - The PHI boundary is enforced in C#, is covered by unit tests, and fails closed rather than emitting an unverified plan. -- Collection is single-instance through the existing lease, so notifications are not multiplied by replica count. +- Collection is single-instance through the existing lease, so the emitted lines are not multiplied by replica count. +- Emission costs a log record rather than a charged metric event, so enabling the feature does not add load to the metrics pipeline. ### Adverse effects - Diagnostics cannot be pulled on demand. Data appears on the collection period, hourly by default, so an incident is served by data that was already being collected rather than by an engineer asking a question and getting an answer straight away. A run window has to be configured in advance. - Settings bind through `IOptions`, so changing them on a running host requires a restart. +- Logs are not aggregated for you. Nothing here arrives as a pre-computed time series, so trend questions need a query over the emitted lines rather than a metric chart. - This watchdog no longer shares the `Watchdog` base class, so it re-implements that class's timer and lease orchestration and will not pick up future improvements to it. That is the cost of keeping configuration out of `dbo.Parameters`, and it should be revisited if the base class itself moves to configuration. - One shared type changed. `WatchdogLease` was constrained to `T : Watchdog` but used its type argument only for `typeof(T).Name`. The constraint was relaxed so that a component which schedules itself can still elect a single replica. It restricted nothing the class actually used, and every existing caller still satisfies it, but it is a change to a shared file and reviewers should confirm they are comfortable with it. - Collection depends on Query Store being enabled and in `READ_WRITE` state on the database. Otherwise the job reports why it cannot collect and does nothing. @@ -56,7 +68,8 @@ We also decided that every setting lives in configuration and none in `dbo.Param ### Neutral effects -- The emitted notifications are contracts that hosts bind to. This repository prescribes no sink. +- The emitted lines are ordinary log records. There is no handler to bind and this repository prescribes no sink. Whatever a deployment already does with FHIR server logs, it does with these. +- The statistics health rows arrive as a JSON array inside one log property rather than as separate columns. A reader has to parse them. That is the trade accepted for batching, and it is affordable because the fields are few and uniform. - The lease continues to write to its own table. That is runtime coordination rather than configuration, and is not part of what this decision removed. ## References @@ -64,4 +77,4 @@ We also decided that every setting lives in configuration and none in `dbo.Param - PR [#5723](https://github.com/microsoft/fhir-server/pull/5723) - `docs/QueryStorePerformanceDiagnostics.md` — design and configuration reference - `docs/arch/adr-2602-database-logging.md` — precedent for diagnostics gathered inside the service -- `docs/arch/adr-2605-metric-emission-rate-limiting.md` — precedent for emission-rate concerns on the metrics pipeline +- `docs/arch/adr-2605-metric-emission-rate-limiting.md` — the emission-rate incident behind the choice of logs over metrics diff --git a/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs index fc2c8eb2cb..026a5e019e 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs @@ -49,6 +49,15 @@ public class QueryStoreDiagnosticsConfiguration /// public int StatisticsHealthCount { get; set; } = 20; + /// + /// Gets or sets the number of table statistics rows packed into each emitted log line. This is not a cap on + /// what is collected — is — only on how many of the collected rows share + /// a line. Rows beyond the batch size are emitted on further lines, each carrying its page number and the + /// total page and row counts. A non-positive value is reported and the default is used, because a batch size + /// cannot pack a row. + /// + public int StatisticsHealthBatchSize { get; set; } = 20; + /// /// Gets or sets the inclusive start of the run window, before which no diagnostics are collected. /// Null, the default, means there is no lower bound and collection can begin immediately. diff --git a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json index 890fd74300..d9f8e43a1b 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json +++ b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json @@ -167,7 +167,8 @@ "MinDurationMilliseconds": 1000, "IncludeQueryPlans": true, "IncludeStatisticsHealth": true, - "StatisticsHealthCount": 20 + "StatisticsHealthCount": 20, + "StatisticsHealthBatchSize": 20 } } }, diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs index fe3b4ccfee..b50cb215bb 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs @@ -7,7 +7,6 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using Medino; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.Core.Configs; @@ -146,7 +145,6 @@ private static QueryStoreDiagnosticsWatchdog CreateWatchdog(ILogger(), logger, - Substitute.For(), Options.Create(configuration)); } diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs index e4685914b3..7e68f39b4a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs @@ -7,7 +7,6 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using Medino; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.Core.Configs; @@ -269,7 +268,6 @@ private static QueryStoreDiagnosticsWatchdog CreateWatchdog( var watchdog = new QueryStoreDiagnosticsWatchdog( Substitute.For(), logger, - Substitute.For(), Options.Create(configuration)); // Constructing the watchdog constructs its WatchdogLease, which logs through this same logger, so diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs new file mode 100644 index 0000000000..8015154901 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs @@ -0,0 +1,271 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.SqlServer.Features.Storage; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +{ + /// + /// Covers the statistics-health batching, which is the one place where what was collected and what reaches a log + /// line can differ. The pagination fields are the only way a reader can tell a short final page from a set that + /// was cut short, and an off-by-one in the page arithmetic silently drops or duplicates rows rather than failing, + /// so every boundary is pinned here. The integration test runs against whatever row count the live schema + /// happens to have and cannot pin any of them. + /// + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public class QueryStoreDiagnosticsStatisticsHealthBatchTests + { + private const int DefaultBatchSize = 20; + + [Fact] + public void GivenExactlyOneFullBatch_WhenEmitting_ThenOneLineCarriesEveryRow() + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, batchSize: 5); + + // Act + watchdog.LogStatisticsHealthBatches(CreateRows(5)); + + // Assert + CapturingLogger.LogEntry entry = Assert.Single(logger.StatisticsHealthEntries); + Assert.Equal(LogLevel.Information, entry.Level); + Assert.Equal(1, entry.Properties["StatisticsHealthPage"]); + Assert.Equal(1, entry.Properties["StatisticsHealthPageCount"]); + Assert.Equal(5, entry.Properties["StatisticsHealthPageRowCount"]); + Assert.Equal(5, entry.Properties["StatisticsHealthRowCount"]); + + // A full batch must not spill into an empty second page, which is what a page count derived by dividing + // and then unconditionally adding one would produce. + AssertRowsAre(entry, 0, 5); + } + + [Fact] + public void GivenAPartialFinalPage_WhenEmitting_ThenTheLastLineReportsOnlyTheRowsItCarries() + { + // Arrange + // Twelve rows at a batch size of five is two full pages and a final page of two. + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, batchSize: 5); + + // Act + watchdog.LogStatisticsHealthBatches(CreateRows(12)); + + // Assert + Assert.Equal(3, logger.StatisticsHealthEntries.Count); + Assert.Equal(new[] { 1, 2, 3 }, logger.StatisticsHealthEntries.Select(entry => (int)entry.Properties["StatisticsHealthPage"])); + Assert.All(logger.StatisticsHealthEntries, entry => Assert.Equal(3, entry.Properties["StatisticsHealthPageCount"])); + + // The whole point of carrying the total: a final page of two out of twelve is complete, and a reader has + // to be able to say so without guessing from the batch size. + Assert.All(logger.StatisticsHealthEntries, entry => Assert.Equal(12, entry.Properties["StatisticsHealthRowCount"])); + Assert.Equal(new[] { 5, 5, 2 }, logger.StatisticsHealthEntries.Select(entry => (int)entry.Properties["StatisticsHealthPageRowCount"])); + + AssertRowsAre(logger.StatisticsHealthEntries[0], 0, 5); + AssertRowsAre(logger.StatisticsHealthEntries[1], 5, 5); + AssertRowsAre(logger.StatisticsHealthEntries[2], 10, 2); + } + + [Fact] + public void GivenMoreRowsThanOneBatch_WhenEmitting_ThenEveryRowIsEmittedExactlyOnceInOrder() + { + // Arrange + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, batchSize: 4); + + // Act + watchdog.LogStatisticsHealthBatches(CreateRows(9)); + + // Assert + Assert.Equal(3, logger.StatisticsHealthEntries.Count); + + // Paging is what makes the batch safe, so the union of the pages has to be the collected set exactly: + // neither a dropped row at a page boundary nor one emitted on two pages. + List emitted = logger.StatisticsHealthEntries + .SelectMany(entry => DeserializeRows(entry).Select(row => row.StatisticsName)) + .ToList(); + Assert.Equal(CreateRows(9).Select(row => row.StatisticsName).ToList(), emitted); + } + + [Fact] + public void GivenANonPositiveBatchSize_WhenEmitting_ThenTheDefaultIsUsedAndTheValueIsReported() + { + // Arrange + // Unlike the counts, a batch size of zero cannot mean "collect nothing": the rows have already been read + // by this point, so degrading to the default is the only option that does not throw away diagnostics. + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, batchSize: 0); + + // Act + watchdog.LogStatisticsHealthBatches(CreateRows(DefaultBatchSize + 1)); + + // Assert + Assert.Equal(2, logger.StatisticsHealthEntries.Count); + Assert.Equal(DefaultBatchSize, logger.StatisticsHealthEntries[0].Properties["StatisticsHealthPageRowCount"]); + Assert.Equal(1, logger.StatisticsHealthEntries[1].Properties["StatisticsHealthPageRowCount"]); + + string warning = Assert.Single(logger.WarningMessages); + Assert.Contains(Format(0), warning, StringComparison.Ordinal); + Assert.Contains(Format(DefaultBatchSize), warning, StringComparison.Ordinal); + } + + [Fact] + public void GivenABatchSizeAboveTheCap_WhenEmitting_ThenItIsClampedAndEveryRowIsStillEmitted() + { + // Arrange + // An unbounded batch would rebuild the single oversized record that is the reason plan XML is never + // batched, so the cap binds. Clamping must page the rows rather than drop the ones past the cap. + const int MaxBatchSize = 64; + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, batchSize: 5000); + + // Act + watchdog.LogStatisticsHealthBatches(CreateRows(MaxBatchSize + 3)); + + // Assert + Assert.Equal(2, logger.StatisticsHealthEntries.Count); + Assert.Equal(MaxBatchSize, logger.StatisticsHealthEntries[0].Properties["StatisticsHealthPageRowCount"]); + Assert.Equal(3, logger.StatisticsHealthEntries[1].Properties["StatisticsHealthPageRowCount"]); + Assert.Equal(MaxBatchSize + 3, logger.StatisticsHealthEntries[0].Properties["StatisticsHealthRowCount"]); + + AssertRowsAre(logger.StatisticsHealthEntries[0], firstRowIndex: 0, expectedRowCount: MaxBatchSize); + AssertRowsAre(logger.StatisticsHealthEntries[1], firstRowIndex: MaxBatchSize, expectedRowCount: 3); + + string warning = Assert.Single(logger.WarningMessages); + Assert.Contains(Format(5000), warning, StringComparison.Ordinal); + Assert.Contains(Format(MaxBatchSize), warning, StringComparison.Ordinal); + } + + [Fact] + public void GivenNoRows_WhenEmitting_ThenNoLineIsEmitted() + { + // Arrange + // The collection summary already reports a count of zero, so an empty page would add nothing except an + // extra page for a reader counting them. + var logger = new CapturingLogger(); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, batchSize: 5); + + // Act + watchdog.LogStatisticsHealthBatches(Array.Empty()); + + // Assert + Assert.Empty(logger.StatisticsHealthEntries); + Assert.Empty(logger.WarningMessages); + } + + private static void AssertRowsAre(CapturingLogger.LogEntry entry, int firstRowIndex, int expectedRowCount) + { + List rows = DeserializeRows(entry); + Assert.Equal(expectedRowCount, rows.Count); + for (int offset = 0; offset < expectedRowCount; offset++) + { + Assert.Equal(RowName(firstRowIndex + offset), rows[offset].StatisticsName); + } + } + + private static List DeserializeRows(CapturingLogger.LogEntry entry) + { + // Deserializing rather than string-matching, because the property is only useful downstream if it really + // is a JSON array of rows. + return JsonSerializer.Deserialize>((string)entry.Properties["StatisticsHealthRows"]); + } + + private static IReadOnlyList CreateRows(int count) + { + return Enumerable.Range(0, count) + .Select(index => new StatisticsHealthDiagnostics + { + SchemaName = "dbo", + TableName = "Resource", + StatisticsName = RowName(index), + Rows = index, + }) + .ToList(); + } + + private static string RowName(int index) => FormattableString.Invariant($"ST_{index}"); + + // The logging infrastructure formats message arguments with the invariant culture, so expected values are + // formatted the same way rather than with whatever culture the test host happens to run under. + private static string Format(int value) => value.ToString(CultureInfo.InvariantCulture); + + private static QueryStoreDiagnosticsWatchdog CreateWatchdog(ILogger logger, int batchSize) + { + var configuration = new WatchdogConfiguration(); + configuration.QueryStoreDiagnostics.Enabled = true; + configuration.QueryStoreDiagnostics.StatisticsHealthBatchSize = batchSize; + + return new QueryStoreDiagnosticsWatchdog( + Substitute.For(), + logger, + Options.Create(configuration)); + } + + /// + /// Records what was logged, at what level, keeping the named properties alongside the formatted message so a + /// test can assert on the values an emitted line carries rather than only on the text it renders to. + /// + private sealed class CapturingLogger : ILogger + { + private readonly List _entries = new List(); + + internal IReadOnlyList WarningMessages => + _entries.Where(entry => entry.Level == LogLevel.Warning).Select(entry => entry.Message).ToList(); + + internal IReadOnlyList StatisticsHealthEntries => + _entries.Where(entry => entry.Message.StartsWith("QueryStoreDiagnosticsWatchdog statistics health.", StringComparison.Ordinal)).ToList(); + + public IDisposable BeginScope(TState state) + where TState : notnull + => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + _entries.Add(new LogEntry(logLevel, formatter(state, exception), state as IReadOnlyList>)); + } + + internal sealed class LogEntry + { + internal LogEntry(LogLevel level, string message, IReadOnlyList> state) + { + Level = level; + Message = message; + + var properties = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair property in state ?? Array.Empty>()) + { + // Assigned rather than added, because a template is free to repeat a placeholder and this + // capture must not throw on a line it merely passes through. + properties[property.Key] = property.Value; + } + + Properties = properties; + } + + internal LogLevel Level { get; } + + internal string Message { get; } + + internal IReadOnlyDictionary Properties { get; } + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs index 93995b307d..32de909df1 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs @@ -5,15 +5,13 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; -using Medino; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.Core.Configs; -using Microsoft.Health.Fhir.Core.Features.Metrics; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; using Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Storage; @@ -36,14 +34,11 @@ public class QueryStoreDiagnosticsWaitStatisticsTests private const int DeadlockErrorNumber = 1205; [Fact] - public async Task GivenAFailingWaitStatisticsRead_WhenCollecting_ThenSlowQueriesArePublishedWithFailedWaitStatus() + public async Task GivenAFailingWaitStatisticsRead_WhenCollecting_ThenSlowQueriesAreLoggedWithFailedWaitStatus() { // Arrange var sqlRetryService = Substitute.For(); - var mediator = Substitute.For(); - var published = new List(); - mediator.When(x => x.PublishAsync(Arg.Any(), Arg.Any())) - .Do(info => published.Add((SlowQueryNotification)info[0])); + var logger = new CapturingLogger(); var slowQuery = new QueryStoreDiagnosticsWatchdog.SlowQueryResult { @@ -83,27 +78,36 @@ public async Task GivenAFailingWaitStatisticsRead_WhenCollecting_ThenSlowQueries .Returns>( _ => throw SqlExceptionFactory.GetSqlException(DeadlockErrorNumber, "Transaction was deadlocked on lock resources.")); - QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(sqlRetryService, mediator); + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(sqlRetryService, logger); // Act await watchdog.CollectDiagnosticsAsync(DateTimeOffset.UtcNow.AddHours(-1), DateTimeOffset.UtcNow, CancellationToken.None); // Assert - // The runtime metrics are the primary signal, so a broken wait read must not suppress them... - SlowQueryNotification notification = Assert.Single(published); - Assert.Equal(slowQuery.QueryId, notification.QueryId); - Assert.Equal(slowQuery.PlanId, notification.PlanId); - Assert.Equal(slowQuery.TotalDurationMilliseconds, notification.TotalDurationMilliseconds); - - // ...and the breakage must be visible on the notification rather than looking like "this plan waited on - // nothing", which is what an Unavailable status would mean. - Assert.Equal(QueryStoreDiagnosticsWatchdog.WaitStatisticsFailedStatus, notification.WaitStatisticsStatus); - Assert.Null(notification.TotalWaitMilliseconds); - Assert.Null(notification.AverageWaitMilliseconds); - Assert.Null(notification.TopWaitCategory); + // The runtime statistics are the primary signal, so a broken wait read must not suppress them... + CapturingLogger.LogEntry entry = Assert.Single(logger.SlowQueryEntries); + Assert.Equal(LogLevel.Information, entry.Level); + Assert.Equal(slowQuery.QueryId, entry.Properties["QueryId"]); + Assert.Equal(slowQuery.PlanId, entry.Properties["PlanId"]); + Assert.Equal(slowQuery.TotalDurationMilliseconds, entry.Properties["TotalDurationMilliseconds"]); + + // ...and the breakage must be visible on the emitted line rather than looking like "this plan waited on + // nothing", which is what an Unavailable status would mean. These are asserted on the structured state + // rather than on the formatted message because a null renders as an empty string once formatted, which + // is indistinguishable from a value that was genuinely reported as empty. + Assert.Equal(QueryStoreDiagnosticsWatchdog.WaitStatisticsFailedStatus, entry.Properties["WaitStatisticsStatus"]); + Assert.Null(entry.Properties["TotalWaitMilliseconds"]); + Assert.Null(entry.Properties["AverageWaitMilliseconds"]); + Assert.Null(entry.Properties["TopWaitCategory"]); + + // The failure itself is still reported, so that a wait read broken for a month is not visible only to + // someone who thought to look at the status field. + Assert.Contains( + logger.WarningMessages, + message => message.Contains("wait statistics could not be read", StringComparison.Ordinal)); } - private static QueryStoreDiagnosticsWatchdog CreateWatchdog(ISqlRetryService sqlRetryService, IMediator mediator) + private static QueryStoreDiagnosticsWatchdog CreateWatchdog(ISqlRetryService sqlRetryService, ILogger logger) { var configuration = new WatchdogConfiguration(); configuration.QueryStoreDiagnostics.Enabled = true; @@ -117,9 +121,59 @@ private static QueryStoreDiagnosticsWatchdog CreateWatchdog(ISqlRetryService sql return new QueryStoreDiagnosticsWatchdog( sqlRetryService, - NullLogger.Instance, - mediator, + logger, Options.Create(configuration)); } + + /// + /// Records what was logged, at what level, keeping the named properties alongside the formatted message so a + /// test can assert on the values an emitted line carries rather than only on the text it renders to. + /// + private sealed class CapturingLogger : ILogger + { + private readonly List _entries = new List(); + + internal IReadOnlyList WarningMessages => + _entries.Where(entry => entry.Level == LogLevel.Warning).Select(entry => entry.Message).ToList(); + + internal IReadOnlyList SlowQueryEntries => + _entries.Where(entry => entry.Message.StartsWith("QueryStoreDiagnosticsWatchdog slow query.", StringComparison.Ordinal)).ToList(); + + public IDisposable BeginScope(TState state) + where TState : notnull + => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + _entries.Add(new LogEntry(logLevel, formatter(state, exception), state as IReadOnlyList>)); + } + + internal sealed class LogEntry + { + internal LogEntry(LogLevel level, string message, IReadOnlyList> state) + { + Level = level; + Message = message; + + var properties = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair property in state ?? Array.Empty>()) + { + // Assigned rather than added, because a template is free to repeat a placeholder and this + // capture must not throw on a line it merely passes through. + properties[property.Key] = property.Value; + } + + Properties = properties; + } + + internal LogLevel Level { get; } + + internal string Message { get; } + + internal IReadOnlyDictionary Properties { get; } + } + } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Metrics/QueryPlanNotification.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanDiagnostics.cs similarity index 76% rename from src/Microsoft.Health.Fhir.Core/Features/Metrics/QueryPlanNotification.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanDiagnostics.cs index 8c107a35ef..921ebd02bf 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Metrics/QueryPlanNotification.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanDiagnostics.cs @@ -5,12 +5,14 @@ using System; -namespace Microsoft.Health.Fhir.Core.Features.Metrics +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { /// - /// Contains a sanitized Query Store execution plan. + /// A sanitized Query Store execution plan, as emitted on a single structured log line. This is a log payload + /// shape rather than a contract anything binds to, so it lives beside the only component that produces it and + /// is internal. /// - public class QueryPlanNotification : IMetricsNotification + internal sealed class QueryPlanDiagnostics { /// /// Gets or sets the Query Store query identifier. @@ -50,18 +52,10 @@ public class QueryPlanNotification : IMetricsNotification public string SanitizationStatus { get; set; } /// - /// Gets or sets the timestamp when the notification was created. + /// Gets or sets the timestamp when the diagnostics were collected. Emitted under a name of its own rather + /// than as Timestamp, because that name collides with the ingestion timestamp the log pipeline + /// supplies for every record. /// public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow; - - /// - /// Gets the FHIR operation associated with this notification. - /// - public string FhirOperation => "query-store-diagnostics"; - - /// - /// Gets the resource type associated with this notification. - /// - public string ResourceType => "System"; } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs index ecc9b1f09c..372c8b7fcb 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs @@ -8,22 +8,24 @@ using System.Data; using System.Diagnostics; using System.Linq; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using EnsureThat; -using Medino; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.Core.Configs; -using Microsoft.Health.Fhir.Core.Features.Metrics; using Microsoft.Health.Fhir.SqlServer.Features.Storage; namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { /// - /// Collects Azure SQL Query Store and statistics diagnostics on a timer and publishes them as metrics - /// notifications. Deliberately does not derive from : that base class seeds and then + /// Collects Azure SQL Query Store and statistics diagnostics on a timer and emits them as structured log lines. + /// Logs rather than metric events: the payload is not metric-shaped — query text and plan XML are unbounded + /// free text and wait categories are high cardinality — and metric events are charged on receipt, which + /// docs/arch/adr-2605-metric-emission-rate-limiting.md records as having throttled a shared metric + /// account. Deliberately does not derive from : that base class seeds and then /// re-reads its period from dbo.Parameters on every start, from a private non-virtual initialization step /// with no override hook, and this feature is configured exclusively through configuration and must write /// nothing to the database. The timer and the lease the base class would have supplied are owned directly @@ -51,6 +53,21 @@ internal sealed class QueryStoreDiagnosticsWatchdog /// The collection interval used when configuration does not supply a usable one. private const double DefaultPeriodSec = 3600; + /// + /// The statistics-health batch size used when configuration does not supply a usable one. Mirrors the class + /// default on . + /// + private const int DefaultStatisticsHealthBatchSize = 20; + + /// + /// The largest number of statistics rows packed into one log line. Batching exists to reduce record count, + /// but an unbounded batch would recreate the single oversized record that is the reason plan XML is not + /// batched at all. A typical serialized row is a little under 400 bytes, so this keeps a full page well + /// inside the budget the feature already applies to its other large fields. + /// Rows above the cap are not dropped; they move to the next page. + /// + private const int MaxStatisticsHealthBatchSize = 64; + /// /// The lease renewal interval. Ten minutes lets the lease be picked up promptly after a replica dies without /// expiring in the middle of a collection. It is an internal coordination knob rather than an operator @@ -144,7 +161,7 @@ ORDER BY queryStorePlan.plan_id;"; private const string WaitStatisticsSql = @" --- Wait statistics are collected independently so unavailable wait capture does not suppress slow-query metrics. +-- Wait statistics are collected independently so unavailable wait capture does not suppress the slow-query lines. ;WITH WaitsByCategory AS ( SELECT @@ -232,7 +249,6 @@ ELSE CONVERT(float, statisticsProperties.modification_counter) / statisticsPrope private readonly QueryStoreDiagnosticsConfiguration _configuration; private readonly ILogger _logger; - private readonly IMediator _mediator; private readonly ISqlRetryService _sqlRetryService; private readonly FhirTimer _fhirTimer; private readonly WatchdogLease _watchdogLease; @@ -251,12 +267,10 @@ ELSE CONVERT(float, statisticsProperties.modification_counter) / statisticsPrope public QueryStoreDiagnosticsWatchdog( ISqlRetryService sqlRetryService, ILogger logger, - IMediator mediator, IOptions watchdogConfiguration) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); - _mediator = EnsureArg.IsNotNull(mediator, nameof(mediator)); _configuration = EnsureArg.IsNotNull(watchdogConfiguration?.Value, nameof(watchdogConfiguration)).QueryStoreDiagnostics; _fhirTimer = new FhirTimer(_logger); _watchdogLease = new WatchdogLease(_sqlRetryService, _logger); @@ -361,7 +375,7 @@ private async Task OnNextTickAsync(CancellationToken cancellationToken) { // The lease is what keeps one collection per period rather than one per replica: without this gate // an eight-instance deployment would issue eight concurrent Query Store scans an hour and emit - // eight copies of every notification. + // eight copies of every diagnostics line. _logger.LogDebug("{WatchdogName}.OnNextTickAsync: skipping because this instance does not hold the lease.", Name); return; } @@ -599,9 +613,9 @@ private async Task RunWorkAsync(CancellationToken cancellationToken) // it in configuration, so "the views you asked me to read do not exist" is always operator-actionable // and permanent, and per-read catches would be more churn than value. // Because the filter spans every read, the missing view can be the last one, after slow queries and - // plans have already been published; the message is therefore deliberately worded to be true of a + // plans have already been emitted; the message is therefore deliberately worded to be true of a // partial tick as well as of one that emitted nothing. - _logger.LogWarning(ex, "QueryStoreDiagnosticsWatchdog: collection was aborted because a required Query Store or statistics view is unavailable. Any diagnostics already emitted during this collection were still published."); + _logger.LogWarning(ex, "QueryStoreDiagnosticsWatchdog: collection was aborted because a required Query Store or statistics view is unavailable. Any diagnostics already emitted during this collection were still logged."); } catch (SqlException ex) when (ex.Number == 229 || ex.Number == 262) { @@ -611,7 +625,7 @@ private async Task RunWorkAsync(CancellationToken cancellationToken) /// /// Runs the collection itself, once the configuration, run-window and Query Store state gates have all - /// passed. Separated from so that the reads and the notifications they produce + /// passed. Separated from so that the reads and the log lines they produce /// are reachable from unit tests, and so that exception handling stays in one place in the caller. /// /// The start of the collection window. @@ -641,8 +655,8 @@ internal async Task CollectDiagnosticsAsync(DateTimeOffset startTime, DateTimeOf waitStatistics.Waits.TryGetValue(slowQuery.PlanId, out var wait); var queryText = slowQuery.QueryText; var queryTextTruncated = queryText.Length > MaxFieldLength; - await _mediator.PublishAsync( - new SlowQueryNotification + LogSlowQuery( + new SlowQueryDiagnostics { QueryId = slowQuery.QueryId, PlanId = slowQuery.PlanId, @@ -663,8 +677,7 @@ await _mediator.PublishAsync( QueryTextLength = queryText.Length, IntervalStart = slowQuery.IntervalStart, IntervalEnd = slowQuery.IntervalEnd, - }, - cancellationToken); + }); } } @@ -675,7 +688,7 @@ await _mediator.PublishAsync( } else if (slowQueries.Count > 0) { - queryPlanCount = await PublishQueryPlansAsync(slowQueries, cancellationToken); + queryPlanCount = await EmitQueryPlansAsync(slowQueries, cancellationToken); } var statisticsHealthCount = 0; @@ -691,7 +704,7 @@ await _mediator.PublishAsync( } else { - statisticsHealthCount = await PublishStatisticsHealthAsync(cancellationToken); + statisticsHealthCount = await EmitStatisticsHealthAsync(cancellationToken); } // A completed tick logs unconditionally, including zero counts: without this, "the watchdog has been @@ -801,15 +814,15 @@ private async Task> GetSlowQueriesAsync(DateTimeO catch (SqlException ex) { // SqlException is caught broadly on purpose: a transient wait-query failure must never abort the tick - // and suppress the runtime metrics, which are the primary signal. The cost of that breadth is that - // timeouts, deadlocks, permission denials and missing views all look alike here, so the failure is - // logged as a warning and surfaced on every notification as WaitStatisticsStatus = Failed. + // and suppress the runtime statistics, which are the primary signal. The cost of that breadth is + // that timeouts, deadlocks, permission denials and missing views all look alike here, so the failure + // is logged as a warning and surfaced on every slow-query line as WaitStatisticsStatus = Failed. _logger.LogWarning(ex, "QueryStoreDiagnosticsWatchdog: Query Store wait statistics could not be read for this collection. Wait fields will be empty."); return (new Dictionary(), true); } } - private async Task PublishQueryPlansAsync(IReadOnlyList slowQueries, CancellationToken cancellationToken) + private async Task EmitQueryPlansAsync(IReadOnlyList slowQueries, CancellationToken cancellationToken) { await using var command = new SqlCommand(QueryPlansSql); command.Parameters.Add("@PlanIds", SqlDbType.NVarChar, -1).Value = string.Join(',', slowQueries.Select(query => query.PlanId)); @@ -824,7 +837,7 @@ private async Task PublishQueryPlansAsync(IReadOnlyList sl // Keyed by plan id and holding the plan XML itself: a plan with no row and a plan whose row carries NULL // XML both yield null here, which is exactly what the sanitizer reports as PlanXmlUnavailable. var plansById = plans.ToDictionary(plan => plan.PlanId, plan => plan.QueryPlanXml); - var publishedPlanCount = 0; + var emittedPlanCount = 0; foreach (var slowQuery in slowQueries) { @@ -833,15 +846,15 @@ private async Task PublishQueryPlansAsync(IReadOnlyList sl if (!string.Equals(sanitizedPlan.Status, QueryPlanSanitizer.SanitizedStatus, StringComparison.Ordinal)) { // Without this, systematic sanitizer breakage looks exactly like "plans are simply unavailable" - // unless a downstream handler happens to surface SanitizationStatus. + // unless whoever is reading the plan lines happens to look at SanitizationStatus. _logger.LogWarning( "QueryStoreDiagnosticsWatchdog: query plan was not emitted because sanitization did not succeed. PlanId={PlanId}, SanitizationStatus={SanitizationStatus}", slowQuery.PlanId, sanitizedPlan.Status); } - await _mediator.PublishAsync( - new QueryPlanNotification + LogQueryPlan( + new QueryPlanDiagnostics { QueryId = slowQuery.QueryId, PlanId = slowQuery.PlanId, @@ -850,31 +863,30 @@ await _mediator.PublishAsync( OriginalQueryPlanLength = sanitizedPlan.OriginalLength, SanitizedQueryPlanLength = sanitizedPlan.SanitizedLength, SanitizationStatus = sanitizedPlan.Status, - }, - cancellationToken); + }); - // A notification is published for every slow query, including the ones with no usable plan, but only - // the ones that carried XML are counted: a count that always equalled the slow-query count would tell - // an operator nothing about whether plans are actually arriving. + // A line is emitted for every slow query, including the ones with no usable plan, but only the ones + // that carried XML are counted: a count that always equalled the slow-query count would tell an + // operator nothing about whether plans are actually arriving. if (sanitizedPlan.Xml != null) { - publishedPlanCount++; + emittedPlanCount++; } } - return publishedPlanCount; + return emittedPlanCount; } - private async Task PublishStatisticsHealthAsync(CancellationToken cancellationToken) + private async Task EmitStatisticsHealthAsync(CancellationToken cancellationToken) { await using var command = new SqlCommand(StatisticsHealthSql); command.Parameters.Add("@Top", SqlDbType.Int).Value = _configuration.StatisticsHealthCount; - // The reader projects straight into the notification contract: an intermediate DTO here would be a + // The reader projects straight into the emitted payload shape: an intermediate DTO here would be a // property-for-property copy of it and nothing else. var statisticsHealth = await _sqlRetryService.ExecuteReaderAsync( command, - reader => new StatisticsHealthNotification + reader => new StatisticsHealthDiagnostics { SchemaName = reader.GetString(0), TableName = reader.GetString(1), @@ -893,12 +905,123 @@ private async Task PublishStatisticsHealthAsync(CancellationToken cancellat "Failed to read statistics health", cancellationToken); - foreach (var statistic in statisticsHealth) + LogStatisticsHealthBatches(statisticsHealth); + + return statisticsHealth.Count; + } + + /// + /// Emits the collected statistics rows as JSON batches, one log line per page. Batched where the slow-query + /// and plan payloads are not because these rows are small, uniform and free of free text, so a batch has a + /// predictable size; a batch of plan XML could not make that promise and would risk one oversized record. + /// The cost of batching is that the rows are a serialized blob rather than queryable columns, which is + /// affordable here precisely because the fields are uniform and cheap to re-parse. Exposed as internal only + /// for unit testing. + /// + /// The collected rows, already capped at StatisticsHealthCount. + internal void LogStatisticsHealthBatches(IReadOnlyList statisticsHealth) + { + if (statisticsHealth.Count == 0) { - await _mediator.PublishAsync(statistic, cancellationToken); + // No line at all rather than an empty page: the collection summary already reports a count of zero, + // and a page carrying nothing would only make the emitted pages harder to count. + return; } - return statisticsHealth.Count; + var batchSize = _configuration.StatisticsHealthBatchSize; + if (batchSize <= 0) + { + // Unlike the counts, this one cannot degrade to collecting nothing — the rows have already been read + // and dropping them would lose diagnostics an operator asked for — so it falls back to the default + // and names the rejected value, the way an unusable PeriodSec does. + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: StatisticsHealthBatchSize is {ConfiguredStatisticsHealthBatchSize}, which cannot pack a row. Falling back to {FallbackStatisticsHealthBatchSize} rows per log line. Configure a positive value to change the batch size.", + batchSize, + DefaultStatisticsHealthBatchSize); + batchSize = DefaultStatisticsHealthBatchSize; + } + else if (batchSize > MaxStatisticsHealthBatchSize) + { + // Clamped rather than honoured, because the failure this prevents is not a smaller page but a single + // record large enough for a sink to truncate or reject, which would lose the rows entirely. Paging + // costs extra lines, and lines are the cheap thing here. + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: StatisticsHealthBatchSize is {ConfiguredStatisticsHealthBatchSize}, which risks a log record too large for the sink to accept. Using {MaxStatisticsHealthBatchSize} rows per log line instead. All rows are still emitted, across more pages.", + batchSize, + MaxStatisticsHealthBatchSize); + batchSize = MaxStatisticsHealthBatchSize; + } + + var pageCount = ((statisticsHealth.Count - 1) / batchSize) + 1; + for (var page = 0; page < pageCount; page++) + { + var pageRows = statisticsHealth.Skip(page * batchSize).Take(batchSize).ToList(); + + // Page number, page count and total row count travel on every line so that a reader can tell a + // partial last page from a set that was cut short by a host that died mid-collection. + _logger.LogInformation( + "QueryStoreDiagnosticsWatchdog statistics health. StatisticsHealthPage={StatisticsHealthPage}, StatisticsHealthPageCount={StatisticsHealthPageCount}, StatisticsHealthPageRowCount={StatisticsHealthPageRowCount}, StatisticsHealthRowCount={StatisticsHealthRowCount}, StatisticsHealthRows={StatisticsHealthRows}", + page + 1, + pageCount, + pageRows.Count, + statisticsHealth.Count, + JsonSerializer.Serialize(pageRows)); + } + } + + /// + /// Emits one slow query as a single structured line. Every field is its own named property rather than a + /// serialized document, so each one lands as a queryable column downstream; at the default of ten rows a + /// tick, a line per row costs nothing worth batching for. + /// + /// The diagnostics to emit. + private void LogSlowQuery(SlowQueryDiagnostics slowQuery) + { + // QueryText is last because it is the one unbounded field on the line, so everything an operator scans + // for is still readable ahead of it. DiagnosticsTimestamp rather than Timestamp, because that name + // collides with the ingestion timestamp the log pipeline supplies for every record. + _logger.LogInformation( + "QueryStoreDiagnosticsWatchdog slow query. QueryId={QueryId}, PlanId={PlanId}, ExecutionCount={ExecutionCount}, TotalDurationMilliseconds={TotalDurationMilliseconds}, AverageDurationMilliseconds={AverageDurationMilliseconds}, MaxDurationMilliseconds={MaxDurationMilliseconds}, TotalCpuMilliseconds={TotalCpuMilliseconds}, AverageCpuMilliseconds={AverageCpuMilliseconds}, TotalLogicalReads={TotalLogicalReads}, AverageLogicalReads={AverageLogicalReads}, TotalWaitMilliseconds={TotalWaitMilliseconds}, AverageWaitMilliseconds={AverageWaitMilliseconds}, TopWaitCategory={TopWaitCategory}, WaitStatisticsStatus={WaitStatisticsStatus}, QueryTextTruncated={QueryTextTruncated}, QueryTextLength={QueryTextLength}, IntervalStart={IntervalStart}, IntervalEnd={IntervalEnd}, DiagnosticsTimestamp={DiagnosticsTimestamp}, QueryText={QueryText}", + slowQuery.QueryId, + slowQuery.PlanId, + slowQuery.ExecutionCount, + slowQuery.TotalDurationMilliseconds, + slowQuery.AverageDurationMilliseconds, + slowQuery.MaxDurationMilliseconds, + slowQuery.TotalCpuMilliseconds, + slowQuery.AverageCpuMilliseconds, + slowQuery.TotalLogicalReads, + slowQuery.AverageLogicalReads, + slowQuery.TotalWaitMilliseconds, + slowQuery.AverageWaitMilliseconds, + slowQuery.TopWaitCategory, + slowQuery.WaitStatisticsStatus, + slowQuery.QueryTextTruncated, + slowQuery.QueryTextLength, + slowQuery.IntervalStart, + slowQuery.IntervalEnd, + slowQuery.Timestamp, + slowQuery.QueryText); + } + + /// + /// Emits one query plan as a single structured line. Not batched with its neighbours: the sanitized XML is + /// capped at the field length rather than being small, so packing several into one record would risk a + /// single oversized log row. + /// + /// The diagnostics to emit. + private void LogQueryPlan(QueryPlanDiagnostics queryPlan) + { + _logger.LogInformation( + "QueryStoreDiagnosticsWatchdog query plan. QueryId={QueryId}, PlanId={PlanId}, SanitizationStatus={SanitizationStatus}, QueryPlanTruncated={QueryPlanTruncated}, OriginalQueryPlanLength={OriginalQueryPlanLength}, SanitizedQueryPlanLength={SanitizedQueryPlanLength}, DiagnosticsTimestamp={DiagnosticsTimestamp}, SanitizedQueryPlan={SanitizedQueryPlan}", + queryPlan.QueryId, + queryPlan.PlanId, + queryPlan.SanitizationStatus, + queryPlan.QueryPlanTruncated, + queryPlan.OriginalQueryPlanLength, + queryPlan.SanitizedQueryPlanLength, + queryPlan.Timestamp, + queryPlan.SanitizedQueryPlan); } /// diff --git a/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SlowQueryDiagnostics.cs similarity index 84% rename from src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SlowQueryDiagnostics.cs index c71abf3e7a..26cc8e1df9 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SlowQueryDiagnostics.cs @@ -5,12 +5,15 @@ using System; -namespace Microsoft.Health.Fhir.Core.Features.Metrics +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { /// - /// Contains aggregated Query Store metrics for a slow query plan. + /// The aggregated Query Store diagnostics for one slow query plan, as emitted on a single structured log line. + /// This is a log payload shape rather than a contract anything binds to: it lives beside the only component that + /// produces it, and is internal because nothing outside this assembly consumes it. Each property becomes its own + /// named log property, so it stays queryable as a column downstream instead of being buried in a serialized blob. /// - public class SlowQueryNotification : IMetricsNotification + internal sealed class SlowQueryDiagnostics { /// /// Gets or sets the Query Store query identifier. @@ -115,18 +118,10 @@ public class SlowQueryNotification : IMetricsNotification public DateTimeOffset IntervalEnd { get; set; } /// - /// Gets or sets the timestamp when the notification was created. + /// Gets or sets the timestamp when the diagnostics were collected. Emitted under a name of its own rather + /// than as Timestamp, because that name collides with the ingestion timestamp the log pipeline + /// supplies for every record. /// public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow; - - /// - /// Gets the FHIR operation associated with this notification. - /// - public string FhirOperation => "query-store-diagnostics"; - - /// - /// Gets the resource type associated with this notification. - /// - public string ResourceType => "System"; } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Metrics/StatisticsHealthNotification.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/StatisticsHealthDiagnostics.cs similarity index 74% rename from src/Microsoft.Health.Fhir.Core/Features/Metrics/StatisticsHealthNotification.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/StatisticsHealthDiagnostics.cs index fd9a450bf5..984d63d54d 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Metrics/StatisticsHealthNotification.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/StatisticsHealthDiagnostics.cs @@ -5,12 +5,16 @@ using System; -namespace Microsoft.Health.Fhir.Core.Features.Metrics +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { /// - /// Contains table statistics health information. + /// Table statistics health for one statistics object. Unlike the slow-query and query-plan payloads, instances of + /// this type are serialized to JSON and carried several to a log line: the rows are small, uniform, and contain no + /// free text, so a batch of them has a predictable size and nothing is lost by giving up per-field columns. + /// Properties are public because only serializes public members; + /// the type itself is internal because nothing outside this assembly consumes it. /// - public class StatisticsHealthNotification : IMetricsNotification + internal sealed class StatisticsHealthDiagnostics { /// /// Gets or sets the schema that owns the table. @@ -73,18 +77,9 @@ public class StatisticsHealthNotification : IMetricsNotification public bool HasFilter { get; set; } /// - /// Gets or sets the timestamp when the notification was created. + /// Gets or sets the timestamp when the diagnostics were collected. Carried on the row rather than only on the + /// log line so that a row stays self-describing once it is lifted out of the batch it was emitted in. /// public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow; - - /// - /// Gets the FHIR operation associated with this notification. - /// - public string FhirOperation => "query-store-diagnostics"; - - /// - /// Gets the resource type associated with this notification. - /// - public string ResourceType => "System"; } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs index 573bd47cb9..d63b477ddf 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs @@ -8,19 +8,17 @@ using System.Data; using System.Diagnostics; using System.Linq; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Medino; using Microsoft.Data.SqlClient; -using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.Core.Configs; -using Microsoft.Health.Fhir.Core.Features.Metrics; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; -using NSubstitute; using Xunit; namespace Microsoft.Health.Fhir.Tests.Integration.Persistence @@ -52,13 +50,11 @@ public QueryStoreDiagnosticsWatchdogTests(SqlServerFhirStorageTestsFixture fixtu } [Fact] - public async Task GivenEnabledWatchdogAndCapturedProbe_WhenRun_ThenPublishesSlowQuerySanitizedPlanAndStatisticsHealth() + public async Task GivenEnabledWatchdogAndCapturedProbe_WhenRun_ThenLogsSlowQuerySanitizedPlanAndStatisticsHealth() { // Arrange - var mediator = Substitute.For(); - var notifications = new List(); - CaptureNotifications(mediator, notifications); - var watchdog = CreateWatchdog(mediator, enabled: true); + var logger = new CapturingLogger(); + var watchdog = CreateWatchdog(logger, enabled: true); string tableName = $"DiagProbe_{Guid.NewGuid():N}"; // Query Store does not preserve comments in query_sql_text, so the probe cannot be tagged with a marker @@ -81,24 +77,23 @@ public async Task GivenEnabledWatchdogAndCapturedProbe_WhenRun_ThenPublishesSlow // Assert // The probe query is grouped by plan_id, and a recompile between executions would produce a second - // plan and therefore a second notification. That is a legitimate outcome, so the assertions are on - // the whole matching set: what must hold is that the executions add up. - List probeNotifications = notifications - .OfType() - .Where(notification => notification.QueryText.Contains(queryAlias, StringComparison.Ordinal)) + // plan and therefore a second log line. That is a legitimate outcome, so the assertions are on the + // whole matching set: what must hold is that the executions add up. + List probeSlowQueries = logger.SlowQueries + .Where(slowQuery => slowQuery.QueryText.Contains(queryAlias, StringComparison.Ordinal)) .ToList(); - Assert.NotEmpty(probeNotifications); + Assert.NotEmpty(probeSlowQueries); // The probe runs a fixed number of times under a GUID alias, so the rollup across Query Store // intervals and plans must sum to exactly that count. - Assert.Equal((long)QueryExecutionCount, probeNotifications.Sum(notification => notification.ExecutionCount)); + Assert.Equal((long)QueryExecutionCount, probeSlowQueries.Sum(slowQuery => slowQuery.ExecutionCount)); // Query Store records microseconds and the contract is milliseconds. The probe runs at MAXDOP 1 and // is timed on the client, so the reported totals must sit inside the wall clock plus a tolerance. A // missing /1000.0 would inflate these by three orders of magnitude and break the upper bound; a // doubly applied one would sink them below MinDurationMilliseconds and the query would never appear. double durationUpperBoundMilliseconds = probeElapsedMilliseconds + ProbeTimingToleranceMilliseconds; - foreach (SlowQueryNotification slowQuery in probeNotifications) + foreach (SlowQueryDiagnostics slowQuery in probeSlowQueries) { Assert.True(slowQuery.QueryId > 0); Assert.True(slowQuery.PlanId > 0); @@ -120,8 +115,9 @@ public async Task GivenEnabledWatchdogAndCapturedProbe_WhenRun_ThenPublishesSlow Assert.Equal(slowQuery.TotalCpuMilliseconds / slowQuery.ExecutionCount, slowQuery.AverageCpuMilliseconds, 3); Assert.True(slowQuery.TotalLogicalReads > 0); - // Wait collection is best-effort and its failure is swallowed so that runtime metrics still - // publish. A status other than Failed is therefore the only proof that the wait SQL executed. + // Wait collection is best-effort and its failure is swallowed so that the runtime statistics + // are still emitted. A status other than Failed is therefore the only proof that the wait SQL + // executed. Assert.Contains( slowQuery.WaitStatisticsStatus, new[] { QueryStoreDiagnosticsWatchdog.WaitStatisticsAvailableStatus, QueryStoreDiagnosticsWatchdog.WaitStatisticsUnavailableStatus }); @@ -136,16 +132,16 @@ public async Task GivenEnabledWatchdogAndCapturedProbe_WhenRun_ThenPublishesSlow Assert.Null(slowQuery.TotalWaitMilliseconds); } - QueryPlanNotification queryPlan = Assert.Single( - notifications.OfType().ToList(), - notification => notification.QueryId == slowQuery.QueryId && notification.PlanId == slowQuery.PlanId); + QueryPlanDiagnostics queryPlan = Assert.Single( + logger.QueryPlans, + plan => plan.QueryId == slowQuery.QueryId && plan.PlanId == slowQuery.PlanId); Assert.Equal(QueryPlanSanitizer.SanitizedStatus, queryPlan.SanitizationStatus); Assert.NotNull(queryPlan.SanitizedQueryPlan); } - AssertStatisticsHealthOrdinals(notifications, tableName); + AssertStatisticsHealthOrdinals(logger, tableName); - foreach (SlowQueryNotification slowQuery in notifications.OfType()) + foreach (SlowQueryDiagnostics slowQuery in logger.SlowQueries) { Assert.DoesNotContain("query_store", slowQuery.QueryText, StringComparison.OrdinalIgnoreCase); Assert.DoesNotContain("dm_db_stats_properties", slowQuery.QueryText, StringComparison.OrdinalIgnoreCase); @@ -158,17 +154,25 @@ public async Task GivenEnabledWatchdogAndCapturedProbe_WhenRun_ThenPublishesSlow } [Fact] - public async Task GivenConfigurationGateDisabled_WhenRun_ThenPublishesNothing() + public async Task GivenConfigurationGateDisabled_WhenRun_ThenNothingIsEmitted() { // Arrange - var mediator = Substitute.For(); - var watchdog = CreateWatchdog(mediator, enabled: false); + var logger = new CapturingLogger(); + var watchdog = CreateWatchdog(logger, enabled: false); // Act await watchdog.RunWorkForTestingAsync(CancellationToken.None); // Assert - Assert.Empty(mediator.ReceivedCalls()); + Assert.Empty(logger.SlowQueries); + Assert.Empty(logger.QueryPlans); + Assert.Empty(logger.StatisticsHealthPages); + + // Asserted alongside the absence of diagnostics, because a tick that collected nothing for some other + // reason — Query Store unavailable, say — would satisfy the assertions above just as well. + Assert.Contains( + logger.Messages, + message => message.Contains("QueryStoreDiagnosticsWatchdog is disabled by configuration", StringComparison.Ordinal)); } [Fact] @@ -179,13 +183,13 @@ public async Task GivenTheWatchdogIsStarted_WhenItRuns_ThenItNeitherSeedsNorRead // untouched. Only a live database can show that: the seeding insert and the period read this watchdog no // longer performs both happened at startup, before the first tick, and neither is visible to a test that // invokes the collection directly. - var mediator = Substitute.For(); + var logger = new CapturingLogger(); // A one-second period keeps the randomized start-up delay inside the test's own budget. The lease's first // acquire attempt is a full lease period away, so no tick of this watchdog reaches a collection here — // which is the point: what is under test is what running it costs the database before it collects // anything. - var watchdog = CreateWatchdog(mediator, enabled: true, periodSec: 1); + var watchdog = CreateWatchdog(logger, enabled: true, periodSec: 1); await using SqlConnection connection = await _fixture.SqlConnectionBuilder.GetSqlConnectionAsync(cancellationToken: CancellationToken.None); await connection.OpenAsync(CancellationToken.None); @@ -214,21 +218,23 @@ public async Task GivenTheWatchdogIsStarted_WhenItRuns_ThenItNeitherSeedsNorRead Assert.Equal(0, await CountWatchdogParametersAsync(connection, CancellationToken.None)); } - private static void AssertStatisticsHealthOrdinals(List notifications, string probeTableName) + private static void AssertStatisticsHealthOrdinals(CapturingLogger logger, string probeTableName) { // Every column of the statistics-health read is taken positionally from a hand-written SELECT list that // no compiler checks, so two same-typed columns could be reordered and every value would silently swap. // The probe table is set up so that its statistics carry values that differ from one another, which pins // those ordinals: a swap would have to preserve every one of these values to go unnoticed. - List statisticsHealth = notifications.OfType().ToList(); + List statisticsHealth = logger.StatisticsHealth; Assert.NotEmpty(statisticsHealth); - StatisticsHealthNotification probeIndexStatistics = Assert.Single( + AssertStatisticsHealthPagination(logger, statisticsHealth.Count); + + StatisticsHealthDiagnostics probeIndexStatistics = Assert.Single( statisticsHealth, - notification => - string.Equals(notification.SchemaName, "dbo", StringComparison.Ordinal) - && string.Equals(notification.TableName, probeTableName, StringComparison.Ordinal) - && string.Equals(notification.StatisticsName, $"PK_{probeTableName}", StringComparison.Ordinal)); + row => + string.Equals(row.SchemaName, "dbo", StringComparison.Ordinal) + && string.Equals(row.TableName, probeTableName, StringComparison.Ordinal) + && string.Equals(row.StatisticsName, $"PK_{probeTableName}", StringComparison.Ordinal)); // The probe table holds ProbeTableRowCount rows at the point its statistics were updated with a full // scan, and exactly ProbeTableModificationCount rows were added afterwards, so every numeric column has @@ -251,12 +257,12 @@ private static void AssertStatisticsHealthOrdinals(List no // A standalone CREATE STATISTICS object reports user_created, and nothing else, which separates that // flag from the three bit columns adjacent to it. - StatisticsHealthNotification probeUserStatistics = Assert.Single( + StatisticsHealthDiagnostics probeUserStatistics = Assert.Single( statisticsHealth, - notification => - string.Equals(notification.SchemaName, "dbo", StringComparison.Ordinal) - && string.Equals(notification.TableName, probeTableName, StringComparison.Ordinal) - && string.Equals(notification.StatisticsName, $"ST_{probeTableName}", StringComparison.Ordinal)); + row => + string.Equals(row.SchemaName, "dbo", StringComparison.Ordinal) + && string.Equals(row.TableName, probeTableName, StringComparison.Ordinal) + && string.Equals(row.StatisticsName, $"ST_{probeTableName}", StringComparison.Ordinal)); Assert.True(probeUserStatistics.IsUserCreated); Assert.False(probeUserStatistics.IsFromIndex); Assert.False(probeUserStatistics.IsAutoCreated); @@ -264,19 +270,32 @@ private static void AssertStatisticsHealthOrdinals(List no // A real FHIR table is asserted on as well, so that the scan is not merely finding the table this test // created. This index is filtered, which is what separates has_filter from is_from_index. - StatisticsHealthNotification filteredIndexStatistics = Assert.Single( + StatisticsHealthDiagnostics filteredIndexStatistics = Assert.Single( statisticsHealth, - notification => - string.Equals(notification.SchemaName, "dbo", StringComparison.Ordinal) - && string.Equals(notification.TableName, "Resource", StringComparison.Ordinal) - && string.Equals(notification.StatisticsName, "IX_Resource_ResourceTypeId_ResourceId", StringComparison.Ordinal)); + row => + string.Equals(row.SchemaName, "dbo", StringComparison.Ordinal) + && string.Equals(row.TableName, "Resource", StringComparison.Ordinal) + && string.Equals(row.StatisticsName, "IX_Resource_ResourceTypeId_ResourceId", StringComparison.Ordinal)); Assert.True(filteredIndexStatistics.HasFilter); Assert.True(filteredIndexStatistics.IsFromIndex); Assert.False(filteredIndexStatistics.IsAutoCreated); Assert.False(filteredIndexStatistics.IsUserCreated); } - private QueryStoreDiagnosticsWatchdog CreateWatchdog(IMediator mediator, bool enabled, double periodSec = 300) + private static void AssertStatisticsHealthPagination(CapturingLogger logger, int expectedRowCount) + { + // The schema carries far more statistics objects than one batch holds, so a live collection exercises + // pagination rather than only the single-page case the unit tests pin. What is asserted here is that the + // pages an operator would read describe the set they actually cover. + IReadOnlyList pages = logger.StatisticsHealthPages; + Assert.NotEmpty(pages); + Assert.Equal(Enumerable.Range(1, pages.Count).ToList(), pages.Select(page => page.PageNumber).ToList()); + Assert.All(pages, page => Assert.Equal(pages.Count, page.PageCount)); + Assert.All(pages, page => Assert.Equal(expectedRowCount, page.RowCount)); + Assert.Equal(expectedRowCount, pages.Sum(page => page.Rows.Count)); + } + + private QueryStoreDiagnosticsWatchdog CreateWatchdog(ILogger logger, bool enabled, double periodSec = 300) { var configuration = new WatchdogConfiguration(); configuration.QueryStoreDiagnostics.Enabled = enabled; @@ -290,23 +309,16 @@ private QueryStoreDiagnosticsWatchdog CreateWatchdog(IMediator mediator, bool en // statistics do not depend on where the staleness ordering happens to place them. configuration.QueryStoreDiagnostics.StatisticsHealthCount = 5000; + // Left at the default so that a live collection, which reads far more statistics objects than one batch + // holds, exercises the pagination rather than packing everything onto a single line. + configuration.QueryStoreDiagnostics.StatisticsHealthBatchSize = new QueryStoreDiagnosticsConfiguration().StatisticsHealthBatchSize; + return new QueryStoreDiagnosticsWatchdog( _fixture.SqlRetryService, - NullLogger.Instance, - mediator, + logger, Options.Create(configuration)); } - private static void CaptureNotifications(IMediator mediator, List notifications) - { - mediator.When(x => x.PublishAsync(Arg.Any(), Arg.Any())) - .Do(info => notifications.Add((SlowQueryNotification)info[0])); - mediator.When(x => x.PublishAsync(Arg.Any(), Arg.Any())) - .Do(info => notifications.Add((QueryPlanNotification)info[0])); - mediator.When(x => x.PublishAsync(Arg.Any(), Arg.Any())) - .Do(info => notifications.Add((StatisticsHealthNotification)info[0])); - } - private static async Task EnableAndVerifyQueryStoreAsync(SqlConnection connection, CancellationToken cancellationToken) { string initialState = await GetQueryStoreStateAsync(connection, cancellationToken); @@ -433,5 +445,184 @@ private static async Task ExecuteNonQueryAsync(SqlConnection connection, string command.CommandText = commandText; await command.ExecuteNonQueryAsync(cancellationToken); } + + /// + /// Reassembles the emitted diagnostics from what the watchdog logged. The named properties of each line are + /// read back into the payload type they were taken from, and the statistics batches are deserialized from + /// the JSON they are carried in, so the assertions are statements about what a reader of these logs would + /// actually be able to recover. + /// + private sealed class CapturingLogger : ILogger + { + private const string SlowQueryPrefix = "QueryStoreDiagnosticsWatchdog slow query."; + private const string QueryPlanPrefix = "QueryStoreDiagnosticsWatchdog query plan."; + private const string StatisticsHealthPrefix = "QueryStoreDiagnosticsWatchdog statistics health."; + + // The lease and the timer log from their own tasks while a collection is running, so the capture has to + // tolerate concurrent writers even though the collection itself is sequential. + private readonly object _syncRoot = new object(); + private readonly List _messages = new List(); + private readonly List _slowQueries = new List(); + private readonly List _queryPlans = new List(); + private readonly List _statisticsHealthPages = new List(); + + internal IReadOnlyList Messages + { + get + { + lock (_syncRoot) + { + return _messages.ToList(); + } + } + } + + internal IReadOnlyList SlowQueries + { + get + { + lock (_syncRoot) + { + return _slowQueries.ToList(); + } + } + } + + internal IReadOnlyList QueryPlans + { + get + { + lock (_syncRoot) + { + return _queryPlans.ToList(); + } + } + } + + internal IReadOnlyList StatisticsHealthPages + { + get + { + lock (_syncRoot) + { + return _statisticsHealthPages.ToList(); + } + } + } + + internal List StatisticsHealth => + StatisticsHealthPages.SelectMany(page => page.Rows).ToList(); + + public IDisposable BeginScope(TState state) + where TState : notnull + => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + string message = formatter(state, exception); + IReadOnlyDictionary properties = ToProperties(state as IReadOnlyList>); + + lock (_syncRoot) + { + _messages.Add(message); + + if (message.StartsWith(SlowQueryPrefix, StringComparison.Ordinal)) + { + _slowQueries.Add(ToSlowQuery(properties)); + } + else if (message.StartsWith(QueryPlanPrefix, StringComparison.Ordinal)) + { + _queryPlans.Add(ToQueryPlan(properties)); + } + else if (message.StartsWith(StatisticsHealthPrefix, StringComparison.Ordinal)) + { + _statisticsHealthPages.Add(ToStatisticsHealthPage(properties)); + } + } + } + + private static IReadOnlyDictionary ToProperties(IReadOnlyList> state) + { + var properties = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair property in state ?? Array.Empty>()) + { + // Assigned rather than added, because a template is free to repeat a placeholder and this + // capture must not throw on a line it merely passes through. + properties[property.Key] = property.Value; + } + + return properties; + } + + private static SlowQueryDiagnostics ToSlowQuery(IReadOnlyDictionary properties) => + new SlowQueryDiagnostics + { + QueryId = (long)properties["QueryId"], + PlanId = (long)properties["PlanId"], + ExecutionCount = (long)properties["ExecutionCount"], + TotalDurationMilliseconds = (double)properties["TotalDurationMilliseconds"], + AverageDurationMilliseconds = (double)properties["AverageDurationMilliseconds"], + MaxDurationMilliseconds = (double)properties["MaxDurationMilliseconds"], + TotalCpuMilliseconds = (double)properties["TotalCpuMilliseconds"], + AverageCpuMilliseconds = (double)properties["AverageCpuMilliseconds"], + TotalLogicalReads = (double)properties["TotalLogicalReads"], + AverageLogicalReads = (double)properties["AverageLogicalReads"], + TotalWaitMilliseconds = (double?)properties["TotalWaitMilliseconds"], + AverageWaitMilliseconds = (double?)properties["AverageWaitMilliseconds"], + TopWaitCategory = (string)properties["TopWaitCategory"], + WaitStatisticsStatus = (string)properties["WaitStatisticsStatus"], + QueryText = (string)properties["QueryText"], + QueryTextTruncated = (bool)properties["QueryTextTruncated"], + QueryTextLength = (int)properties["QueryTextLength"], + IntervalStart = (DateTimeOffset)properties["IntervalStart"], + IntervalEnd = (DateTimeOffset)properties["IntervalEnd"], + Timestamp = (DateTimeOffset)properties["DiagnosticsTimestamp"], + }; + + private static QueryPlanDiagnostics ToQueryPlan(IReadOnlyDictionary properties) => + new QueryPlanDiagnostics + { + QueryId = (long)properties["QueryId"], + PlanId = (long)properties["PlanId"], + SanitizedQueryPlan = (string)properties["SanitizedQueryPlan"], + QueryPlanTruncated = (bool)properties["QueryPlanTruncated"], + OriginalQueryPlanLength = (int)properties["OriginalQueryPlanLength"], + SanitizedQueryPlanLength = (int)properties["SanitizedQueryPlanLength"], + SanitizationStatus = (string)properties["SanitizationStatus"], + Timestamp = (DateTimeOffset)properties["DiagnosticsTimestamp"], + }; + + private static StatisticsHealthPage ToStatisticsHealthPage(IReadOnlyDictionary properties) => + new StatisticsHealthPage( + (int)properties["StatisticsHealthPage"], + (int)properties["StatisticsHealthPageCount"], + (int)properties["StatisticsHealthRowCount"], + JsonSerializer.Deserialize>((string)properties["StatisticsHealthRows"])); + + internal sealed class StatisticsHealthPage + { + internal StatisticsHealthPage(int pageNumber, int pageCount, int rowCount, List rows) + { + PageNumber = pageNumber; + PageCount = pageCount; + RowCount = rowCount; + Rows = rows; + } + + /// Gets the 1-based page number this line carried. + internal int PageNumber { get; } + + /// Gets the number of pages the collection was emitted across. + internal int PageCount { get; } + + /// Gets the total number of rows collected, across every page. + internal int RowCount { get; } + + /// Gets the rows carried on this page, deserialized from the batch property. + internal List Rows { get; } + } + } } } From dd5d576b26a5560ecba4ec8688e8a5d0b4af3bfc Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 26 Aug 2026 18:26:30 +0000 Subject: [PATCH 19/20] refactor: group Query Store diagnostics files into a feature folder The Watchdogs folder followed an implicit one-file-per-watchdog convention. This feature had added six flat files to it, becoming ~40% of the folder and burying the six sibling watchdogs. Move the feature's files into Features/Watchdogs/QueryStoreDiagnostics/, with the three log payload shapes under a Models/ subfolder, and mirror the same structure in the unit test project. Namespaces follow the folder layout, matching the convention used by Schema/Model, Search/Expressions, Storage/TvpRowGeneration and Operations/Import. All moves are pure relocations - no behaviour changes. Both projects are SDK-style with automatic globbing, so no csproj edits are needed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 2 +- .../QueryPlanSanitizationResultTests.cs | 4 +++- .../{ => QueryStoreDiagnostics}/QueryPlanSanitizerTests.cs | 4 +++- .../QueryStoreDiagnosticsConfigurationOnlyTests.cs | 4 +++- .../QueryStoreDiagnosticsPeriodTests.cs | 4 +++- .../QueryStoreDiagnosticsRunWindowTests.cs | 4 +++- .../QueryStoreDiagnosticsStatisticsHealthBatchTests.cs | 4 +++- .../QueryStoreDiagnosticsWaitStatisticsTests.cs | 4 +++- .../QueryStoreReadonlyReasonTests.cs | 4 +++- .../Models}/QueryPlanDiagnostics.cs | 2 +- .../Models}/SlowQueryDiagnostics.cs | 2 +- .../Models}/StatisticsHealthDiagnostics.cs | 2 +- .../QueryPlanSanitizationResult.cs | 2 +- .../{ => QueryStoreDiagnostics}/QueryPlanSanitizer.cs | 2 +- .../QueryStoreDiagnosticsWatchdog.cs | 3 ++- .../Features/Watchdogs/WatchdogsBackgroundService.cs | 1 + .../FhirServerBuilderSqlServerRegistrationExtensions.cs | 1 + .../Persistence/QueryStoreDiagnosticsWatchdogTests.cs | 2 ++ 18 files changed, 36 insertions(+), 15 deletions(-) rename src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryPlanSanitizationResultTests.cs (95%) rename src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryPlanSanitizerTests.cs (98%) rename src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryStoreDiagnosticsConfigurationOnlyTests.cs (94%) rename src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryStoreDiagnosticsPeriodTests.cs (97%) rename src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryStoreDiagnosticsRunWindowTests.cs (98%) rename src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs (98%) rename src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryStoreDiagnosticsWaitStatisticsTests.cs (97%) rename src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryStoreReadonlyReasonTests.cs (95%) rename src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/{ => QueryStoreDiagnostics/Models}/QueryPlanDiagnostics.cs (96%) rename src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/{ => QueryStoreDiagnostics/Models}/SlowQueryDiagnostics.cs (98%) rename src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/{ => QueryStoreDiagnostics/Models}/StatisticsHealthDiagnostics.cs (97%) rename src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryPlanSanitizationResult.cs (98%) rename src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryPlanSanitizer.cs (98%) rename src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/{ => QueryStoreDiagnostics}/QueryStoreDiagnosticsWatchdog.cs (99%) diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 4d1c9a3544..60eefe6d7b 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -129,7 +129,7 @@ Everything this feature produces is a **structured log record**, written through All three payload lines are emitted at **information** level. The warnings this feature raises — misconfiguration, an unavailable Query Store, a failed wait read, a plan that would not sanitize — remain at **warning** level and are unaffected by how the payload is emitted. -The payload shapes are `SlowQueryDiagnostics`, `QueryPlanDiagnostics`, and `StatisticsHealthDiagnostics`, in `Microsoft.Health.Fhir.SqlServer/Features/Watchdogs`. They are `internal` and live beside the watchdog because they are log payload shapes, not contracts another assembly binds to. +The payload shapes are `SlowQueryDiagnostics`, `QueryPlanDiagnostics`, and `StatisticsHealthDiagnostics`, in `Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/Models`. They are `internal` and live beside the watchdog because they are log payload shapes, not contracts another assembly binds to. ### Why logs rather than metrics diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizationResultTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizationResultTests.cs similarity index 95% rename from src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizationResultTests.cs rename to src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizationResultTests.cs index b30ae560cd..70c64603ba 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizationResultTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizationResultTests.cs @@ -7,11 +7,13 @@ using System.Linq; using System.Reflection; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using Xunit; -namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Operations)] diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizerTests.cs similarity index 98% rename from src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs rename to src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizerTests.cs index 48c85a285a..b4a3c65ef9 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryPlanSanitizerTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizerTests.cs @@ -6,11 +6,13 @@ using System; using System.Xml.Linq; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using Xunit; -namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Operations)] diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsConfigurationOnlyTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationOnlyTests.cs similarity index 94% rename from src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsConfigurationOnlyTests.cs rename to src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationOnlyTests.cs index 4a4ed079c2..e4e486c40f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsConfigurationOnlyTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationOnlyTests.cs @@ -7,11 +7,13 @@ using System.Linq; using System.Reflection; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using Xunit; -namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics { /// /// Pins the property the feature is required to have: it is configured entirely through configuration and never diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsPeriodTests.cs similarity index 97% rename from src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs rename to src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsPeriodTests.cs index b50cb215bb..6bcb5767b6 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsPeriodTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsPeriodTests.cs @@ -12,12 +12,14 @@ using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics { /// /// Covers the collection period, which is the one setting whose misconfiguration reaches outside this feature: diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsRunWindowTests.cs similarity index 98% rename from src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs rename to src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsRunWindowTests.cs index 7e68f39b4a..716dcb5539 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsRunWindowTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsRunWindowTests.cs @@ -12,12 +12,14 @@ using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics { /// /// Covers the optional run window, which decides whether a tick collects anything at all. Every boundary case is diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs similarity index 98% rename from src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs rename to src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs index 8015154901..9e7b98de83 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsStatisticsHealthBatchTests.cs @@ -13,12 +13,14 @@ using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics { /// /// Covers the statistics-health batching, which is the one place where what was collected and what reaches a log diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsWaitStatisticsTests.cs similarity index 97% rename from src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs rename to src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsWaitStatisticsTests.cs index 32de909df1..23220bc15b 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnosticsWaitStatisticsTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsWaitStatisticsTests.cs @@ -14,13 +14,15 @@ using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; using Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Storage; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; -namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics { /// /// Covers the branch that reports a broken wait collection. The integration test can only assert that the status diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreReadonlyReasonTests.cs similarity index 95% rename from src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs rename to src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreReadonlyReasonTests.cs index d07e257c9d..e60ef685b5 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreReadonlyReasonTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreReadonlyReasonTests.cs @@ -6,11 +6,13 @@ using System; using System.Globalization; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using Xunit; -namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Operations)] diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanDiagnostics.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/Models/QueryPlanDiagnostics.cs similarity index 96% rename from src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanDiagnostics.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/Models/QueryPlanDiagnostics.cs index 921ebd02bf..78220230fb 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanDiagnostics.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/Models/QueryPlanDiagnostics.cs @@ -5,7 +5,7 @@ using System; -namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models { /// /// A sanitized Query Store execution plan, as emitted on a single structured log line. This is a log payload diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SlowQueryDiagnostics.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/Models/SlowQueryDiagnostics.cs similarity index 98% rename from src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SlowQueryDiagnostics.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/Models/SlowQueryDiagnostics.cs index 26cc8e1df9..5888d8b924 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SlowQueryDiagnostics.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/Models/SlowQueryDiagnostics.cs @@ -5,7 +5,7 @@ using System; -namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models { /// /// The aggregated Query Store diagnostics for one slow query plan, as emitted on a single structured log line. diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/StatisticsHealthDiagnostics.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/Models/StatisticsHealthDiagnostics.cs similarity index 97% rename from src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/StatisticsHealthDiagnostics.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/Models/StatisticsHealthDiagnostics.cs index 984d63d54d..627670ce27 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/StatisticsHealthDiagnostics.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/Models/StatisticsHealthDiagnostics.cs @@ -5,7 +5,7 @@ using System; -namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models { /// /// Table statistics health for one statistics object. Unlike the slow-query and query-plan payloads, instances of diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizationResult.cs similarity index 98% rename from src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizationResult.cs index 1a1e6a8e61..2656b06caa 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizationResult.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizationResult.cs @@ -5,7 +5,7 @@ using EnsureThat; -namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics { /// /// Outcome of Showplan sanitization. diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizer.cs similarity index 98% rename from src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizer.cs index 640bb5e898..fa819b99d4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryPlanSanitizer.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryPlanSanitizer.cs @@ -9,7 +9,7 @@ using System.Xml; using System.Xml.Linq; -namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics { /// /// Removes Showplan parameter metadata, which can carry literal values taken from patient data, and verifies diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsWatchdog.cs similarity index 99% rename from src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsWatchdog.cs index 372c8b7fcb..558f721b30 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnosticsWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsWatchdog.cs @@ -17,8 +17,9 @@ using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.SqlServer.Features.Storage; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; -namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics { /// /// Collects Azure SQL Query Store and statistics diagnostics on a timer and emits them as structured log lines. diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index c349198ba3..178a30d3fc 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -17,6 +17,7 @@ using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Messages.Search; using Microsoft.Health.Fhir.Core.Messages.Storage; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { diff --git a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs index 4778fdb462..17c3bc5ebf 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs @@ -37,6 +37,7 @@ using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.SqlServer.Features.Storage.Registry; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; using Microsoft.Health.Fhir.SqlServer.Registration; using Microsoft.Health.JobManagement; using Microsoft.Health.SqlServer.Api.Registration; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs index d63b477ddf..55df85f98e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs @@ -16,6 +16,8 @@ using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; From 1ae6d8d634bfe63d362ce92b89d8c85bb3b3e6ba Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 27 Aug 2026 00:10:40 +0000 Subject: [PATCH 20/20] refactor: revert WatchdogLease constraint, keep configuration authoritative WatchdogLease had its `where T : Watchdog` constraint relaxed so the Query Store diagnostics watchdog could elect a single replica while scheduling itself. Revert that shared file to origin/main and derive from Watchdog like every other watchdog. Deriving from the base class reintroduces the two dbo.Parameters rows it seeds, which is acceptable provided configuration can still set them. By default it cannot: dbo.Parameters is declared WITH (IGNORE_DUP_KEY = ON), so on a database that already holds the rows the seeding INSERT is a silent no-op, and InitParamsAsync then reads the stale stored value back over the configured one. Verified against SQL Server: seeding 3600 then 60 for the same Id reports "Duplicate key was ignored. (0 rows affected)" and leaves 3600 in place. A fresh database honours the environment variable while an upgraded one quietly does not. Override InitAdditionalParamsAsync -- the one initialization step the base class makes virtual, which runs after that read-back and before the timer is built -- to UPDATE both rows to the configured values and re-assign the properties. UPDATE rather than INSERT precisely because IGNORE_DUP_KEY would make a re-INSERT a no-op. Configuration now wins on every database and the rows are a readable mirror of it rather than an input to it. The UPDATE is wrapped in a catch that logs and continues: it runs outside FhirTimer's per-tick catch, where a throw would fault this watchdog's task and cause WatchdogsBackgroundService to cancel every other watchdog with it. The property assignments sit outside the try, so the functional guarantee does not depend on the cosmetic one. LeasePeriodSec becomes a real configuration setting rather than a hard-coded const, since the base class stores it and every stored value must be settable from configuration. The unit test that pinned the old invariant is rewritten to pin the new one, and the integration test that asserted zero parameter rows now pre-seeds stale rows and asserts they are reconciled. Confirmed to be a real guard by mutation: removing the UPDATE fails it with 999999 vs 1. WatchdogLease.cs, Watchdog.cs and FhirTimer.cs are byte-identical to origin/main. Solution builds 0 warnings; 98 unit and 3 integration tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QueryStorePerformanceDiagnostics.md | 28 +- ...608-query-store-performance-diagnostics.md | 13 +- .../QueryStoreDiagnosticsConfiguration.cs | 8 + .../appsettings.json | 1 + ...yStoreDiagnosticsConfigurationOnlyTests.cs | 60 ----- ...DiagnosticsConfigurationPrecedenceTests.cs | 103 ++++++++ .../QueryStoreDiagnosticsLeasePeriodTests.cs | 120 +++++++++ .../QueryStoreDiagnosticsWatchdog.cs | 245 ++++++++++-------- .../Features/Watchdogs/WatchdogLease.cs | 5 +- .../QueryStoreDiagnosticsWatchdogTests.cs | 86 ++++-- 10 files changed, 465 insertions(+), 204 deletions(-) delete mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationOnlyTests.cs create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationPrecedenceTests.cs create mode 100644 src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsLeasePeriodTests.cs diff --git a/docs/QueryStorePerformanceDiagnostics.md b/docs/QueryStorePerformanceDiagnostics.md index 60eefe6d7b..de95bbb247 100644 --- a/docs/QueryStorePerformanceDiagnostics.md +++ b/docs/QueryStorePerformanceDiagnostics.md @@ -51,7 +51,9 @@ The critical property is that **nothing new connects inbound to the database**. - **A managed background timer.** `FhirTimer` supplies the randomized start-up stagger that keeps replicas from collecting on the same second, and catches whatever a tick throws so that a failed collection costs one tick rather than the process. - **A precedent for emitting SQL telemetry.** `GeoReplicationLagWatchdog` reads a SQL view on a timer and emits what it finds. This feature has the same shape, but emits **logs rather than metric notifications**; see [Why logs rather than metrics](#why-logs-rather-than-metrics). A single lag figure is metric-shaped, and query text and plan XML are not. -**It deliberately does not derive from `Watchdog`.** That base class inserts `{Name}.PeriodSec` and `{Name}.LeasePeriodSec` into `dbo.Parameters` on every start and then reads the period back **over** the configured value, from a private, non-virtual initialization step with no override hook. This feature is configured exclusively from configuration and writes nothing to the database, so it owns a `FhirTimer` and a `WatchdogLease` directly and reproduces the rest of what the base class did — the lease-holder gate, the capped randomized stagger, and the per-tick timing line — in its own `ExecuteAsync` and tick handler. `WatchdogLease` uses its type argument only to derive the lease resource name from `typeof(T).Name`; its former `T : Watchdog` constraint restricted nothing it actually used, so it was relaxed to admit a self-scheduling component. Every other caller passes a `Watchdog` and is unaffected, and the timer and base class are used unmodified. +**It derives from `Watchdog` like every other watchdog, and keeps configuration authoritative through the one hook the base class provides.** The base class inserts `{Name}.PeriodSec` and `{Name}.LeasePeriodSec` into `dbo.Parameters` on every start and then reads both back **over** the configured values, from a private, non-virtual initialization step. That read-back is not harmless: `dbo.Parameters` is declared `PRIMARY KEY CLUSTERED (Id) WITH (IGNORE_DUP_KEY = ON)`, so on a database that already holds those rows the seeding `INSERT` is a silent no-op — it neither inserts nor errors — and the stored value therefore wins over the environment variable. Verified directly against SQL Server: inserting `3600` and then `60` for the same `Id` reports *"Duplicate key was ignored. (0 rows affected)"* and leaves `3600` in place. + +The base class does expose one overridable step, `InitAdditionalParamsAsync`, which runs after that read-back and before the timer is constructed. This watchdog overrides it to `UPDATE` both rows to the configured values and re-assign the two properties, so configuration wins on every database and the rows stay an accurate mirror of the deployment's settings rather than a stale copy that silently contradicts them. An `UPDATE` is used rather than an `INSERT` precisely because `IGNORE_DUP_KEY` would make a re-`INSERT` a no-op. Deriving from the base class means the lease-holder gate, the capped randomized stagger, and the per-tick timing line come from shared code rather than being reproduced here, and no shared file is modified for this feature. ### Enablement @@ -61,11 +63,13 @@ The feature is **off by default** and is gated by one switch, in configuration. | --- | --- | --- | | `FhirServer:Watchdog:QueryStoreDiagnostics:Enabled` | Host configuration | When false the watchdog is never started by `WatchdogsBackgroundService`, and no Query Store read occurs. | -**All configuration for this feature lives in configuration, and the feature writes none of it to the database.** There is no row to seed, arm, or update: no `IsEnabled` row, no `PeriodSec` row, no `LeasePeriodSec` row. Turning the feature on, tuning it, and turning it off are configuration changes plus a restart — no `UPDATE` against a live database, and no possibility of a database holding a value that disagrees with the deployment's configuration. +**Every setting for this feature is set in configuration, and configuration always wins.** There is no row to seed or arm before the feature will run: no `IsEnabled` row, and nothing an operator has to write by hand. Turning the feature on, tuning it, and turning it off are configuration changes plus a restart, with no `UPDATE` against a live database required of anyone. + +The base class does keep `{Name}.PeriodSec` and `{Name}.LeasePeriodSec` in `dbo.Parameters`, and this watchdog reconciles both rows to the configured values during initialization, so a database can never hold a value that disagrees with the deployment's configuration. The rows are a readable mirror of what the service is running with, not an input to it — editing one changes nothing, because the next start overwrites it. The watchdog also re-reads `Enabled` at the start of every tick and returns without collecting when it is false. That check cannot be reached with the value false as the host is wired today: `WatchdogsBackgroundService` gates startup on the same configuration snapshot, and `IOptions` does not reload in place. It is kept because the watchdog is registered `AsSelf` and this is the only place the opt-out is enforced at the unit of work, so a future call site that executes a collection directly cannot bypass it. The cost is one boolean read per period. -The only database row this feature causes to exist is its **lease**, in `dbo.WatchdogLeases` through `dbo.AcquireWatchdogLease`. That is what stops every replica collecting and emitting the same diagnostics every period; it holds no configuration and is the same mechanism every other watchdog uses. That shared stored procedure does consult `dbo.Parameters` for the fleet-wide watchdog lease-holder include and exclude patterns, which is noted under [Configuration](#configuration): it is shared framework behaviour that this feature neither sets nor reads for itself. +This feature causes three database rows to exist. Two are its `dbo.Parameters` rows, described above, which mirror configuration rather than drive it. The third is its **lease**, in `dbo.WatchdogLeases` through `dbo.AcquireWatchdogLease`. The lease is what stops every replica collecting and emitting the same diagnostics every period; it holds no configuration and is the same mechanism every other watchdog uses. That shared stored procedure does consult `dbo.Parameters` for the fleet-wide watchdog lease-holder include and exclude patterns, which is noted under [Configuration](#configuration): it is shared framework behaviour that this feature neither sets nor reads for itself. ### Configuration @@ -75,6 +79,7 @@ The only database row this feature causes to exist is its **lease**, in `dbo.Wat | --- | --- | --- | | `Enabled` | `false` | The gate described above. | | `PeriodSec` | `3600` | Interval between collections, and — clamped to `[60, 86400]` seconds — the Query Store lookback window each collection covers. A non-positive or non-finite value is rejected with a warning and the `3600` default is used, because `PeriodicTimer` would otherwise throw and fault every watchdog in the process. | +| `LeasePeriodSec` | `600` | How long the single-runner lease is held before it must be renewed. Long enough that a collection cannot outlive it, short enough that a dead replica's lease is picked up promptly. A non-positive or non-finite value is rejected with a warning and the `600` default is used. Most deployments should leave this alone; it is on the configuration surface because the base class stores it in `dbo.Parameters`, and every stored value must be settable from configuration. | | `SlowQueryCount` | `10` | Number of slow plans to report per tick. | | `MinDurationMilliseconds` | `1000` | Minimum weighted average duration for a plan to be reported. A negative value is treated as `0` — which makes every query qualify — and warns. | | `IncludeQueryPlans` | `true` | Whether sanitized Showplan XML is emitted. | @@ -88,7 +93,7 @@ The lookback window is `PeriodSec` clamped to `[60, 86400]` seconds, so the coll The watchdog warns on every tick where the clamp changed the value, naming the effective window and what the clamp costs, and pointing at `FhirServer:Watchdog:QueryStoreDiagnostics:PeriodSec` as the thing to change. The tick interval itself is **not** clamped, so whenever the clamp bites the interval and the window it covers are permanently decoupled: a period above the cap leaves the excess of every interval unexamined, and one below the floor makes consecutive collections overlap and re-report the same plans. -**Every setting takes effect on restart, on every database.** All binding is through `IOptions` rather than `IOptionsMonitor`, so nothing reloads in place: editing configuration on a running host changes nothing until it restarts. `PeriodSec` is read once, at construction, because it is handed to the timer for the life of the process; the rest are read from configuration on each tick, which makes no observable difference while the values cannot change underneath. None of them is read from or written to the database, so a deployment behaves identically against a database it created a minute ago and one it has been running against for a year — there is no first-run state for its settings to diverge from. +**Every setting takes effect on restart, on every database.** All binding is through `IOptions` rather than `IOptionsMonitor`, so nothing reloads in place: editing configuration on a running host changes nothing until it restarts. `PeriodSec` and `LeasePeriodSec` are read once, at construction, because they are handed to the timer and the lease for the life of the process; the rest are read from configuration on each tick, which makes no observable difference while the values cannot change underneath. `PeriodSec` and `LeasePeriodSec` are stored in `dbo.Parameters` but never taken from it — initialization overwrites both rows with the configured values — so a deployment behaves identically against a database it created a minute ago and one it has been running against for a year. There is no first-run state for its settings to diverge from. One piece of pre-existing database state can still suppress collection, and it belongs to the shared lease rather than to this feature: `dbo.AcquireWatchdogLease` honours `WatchdogLeaseHolderIncludePattern` and `WatchdogLeaseHolderExcludePattern` rows in `dbo.Parameters`, plus the per-watchdog `...For` variants — where the name is `QueryStoreDiagnosticsWatchdog`. A worker excluded by such a row never becomes lease holder, so its ticks skip and nothing is collected. That applies identically to every watchdog in the process and is not something this feature sets, seeds, or reads itself, but it is the one thing to check on a long-lived database when the feature is enabled and silent. @@ -249,7 +254,9 @@ Clamping never drops rows. A batch size above the cap simply produces more pages The missing-view handler spans the whole collection rather than each individual read, so the aborting read can be the last one, after slow queries and plans have already been emitted. Its warning is worded to hold in that case too: it reports that collection was aborted and that whatever had already been emitted was still logged, rather than claiming that nothing was collected. -That containment covers **per-tick collection**, which is where all of this feature's own database work happens: `FhirTimer` catches whatever a tick throws and keeps ticking, so a failed collection costs one tick. The lease renewal is the only other database call, and it runs on the lease's own `FhirTimer` with the same per-tick catch. There is no initialization step left to fail outside either catch. Watchdogs that derive from `Watchdog` do have one — `ExecuteAsync` awaits `InitParamsAsync`, which seeds `dbo.Parameters`, *before* and *outside* the per-tick catch, so a throw there faults the watchdog task and `WatchdogsBackgroundService` cancels the rest — and not deriving from it removes that failure mode here along with the writes. +That containment covers **per-tick collection**, which is where all of this feature's own collection work happens: `FhirTimer` catches whatever a tick throws and keeps ticking, so a failed collection costs one tick. The lease renewal runs on the lease's own `FhirTimer` with the same per-tick catch. + +Deriving from `Watchdog` does add one step outside those catches: `ExecuteAsync` awaits `InitParamsAsync`, which seeds `dbo.Parameters` and then calls this feature's `InitAdditionalParamsAsync`, *before* and *outside* the per-tick catch — so a throw there faults the watchdog task and `WatchdogsBackgroundService` cancels the rest. That exposure is shared with every other watchdog, but this feature narrows its own contribution to it: the reconciling `UPDATE` is wrapped in a catch that logs a warning and continues, and the assignments that make configuration authoritative run outside that try. A diagnostics feature is not permitted to fail the transaction and cleanup watchdogs over a row it writes only so the table reads truthfully, and collection is unaffected when the update fails. What remains outside the catch is the period itself: `PeriodicTimer` throws on a non-positive or non-finite interval, and that throw would happen before the first tick. This is why the configured `PeriodSec` is validated at construction and replaced with the default rather than passed through, and why the run-window check skips a tick rather than ending the timer. @@ -367,11 +374,12 @@ Secondary benefits of the change: 1. With Query Store enabled and a deliberately slow query executed, a slow-query line is emitted carrying a matching `QueryId`/`PlanId`. 2. A query-plan line is emitted for that plan with status `Sanitized`. 3. Statistics-health rows are emitted for user tables, batched and paginated, with page numbers and totals that account for every collected row. -4. The watchdog performs no work when the configuration gate is off, and starting it creates no `dbo.Parameters` row and reads none. -5. A non-`READ_WRITE` Query Store state is handled without error and without emission. -6. Wait-statistic unavailability degrades to null wait fields while runtime results are still emitted, and `WaitStatisticsStatus` reports which of the three outcomes occurred. -7. The watchdog does not report its own Query Store queries. -8. Emitted content is asserted, not merely emission: the execution count matches the number of probe executions, and duration and CPU sit in a plausible millisecond range, which is what catches a regression in the weighted rollup or in the microsecond-to-millisecond conversion. +4. The watchdog performs no work when the configuration gate is off. +5. Pre-existing `dbo.Parameters` rows holding stale values are reconciled to the configured values during initialization, so configuration wins on a database that already holds them. +6. A non-`READ_WRITE` Query Store state is handled without error and without emission. +7. Wait-statistic unavailability degrades to null wait fields while runtime results are still emitted, and `WaitStatisticsStatus` reports which of the three outcomes occurred. +8. The watchdog does not report its own Query Store queries. +9. Emitted content is asserted, not merely emission: the execution count matches the number of probe executions, and duration and CPU sit in a plausible millisecond range, which is what catches a regression in the weighted rollup or in the microsecond-to-millisecond conversion. ## References diff --git a/docs/arch/adr-2608-query-store-performance-diagnostics.md b/docs/arch/adr-2608-query-store-performance-diagnostics.md index f9e4cf58f0..313a345eba 100644 --- a/docs/arch/adr-2608-query-store-performance-diagnostics.md +++ b/docs/arch/adr-2608-query-store-performance-diagnostics.md @@ -44,7 +44,11 @@ Logs are also the cheap place to start. If a specific number later turns out to The statistics health rows are the one payload that is batched. They are small, uniform, and free of free text, so several of them fit on one line as a JSON array without any risk of an oversized record. Each line carries its page number, the page count, and the total row count, so a reader can tell a short final page from a set that was cut short. The slow query and plan lines are not batched. Each field there is its own named log property, which is what keeps it queryable as a column, and plan XML is large enough that batching it would risk a single oversized record. -We also decided that every setting lives in configuration and none in `dbo.Parameters`. The first iteration followed the existing watchdog convention of keeping runtime values in that table. That meant an operator had to run an `UPDATE` to arm the feature, and the collection period was read back from the database over the top of the configured value. Both work against the goal: a feature whose purpose is to remove the need for database access should not require a write to the database to switch on, and configuration that is silently overridden by a stored row is not really configuration. Honoring this meant the watchdog could no longer derive from the shared `Watchdog` base class, whose initialization writes those rows and reads them back from a private, non-virtual step with no override hook. The distributed lease that stops every replica collecting the same data was kept. +We also decided that every setting is set in configuration, and that configuration always wins over `dbo.Parameters`. The first iteration followed the existing watchdog convention of keeping runtime values in that table. That meant an operator had to run an `UPDATE` to arm the feature, and the collection period was read back from the database over the top of the configured value. Both work against the goal: a feature whose purpose is to remove the need for database access should not require a write to the database to switch on, and configuration that is silently overridden by a stored row is not really configuration. + +That second problem is sharper than it first appears. `dbo.Parameters` is declared `WITH (IGNORE_DUP_KEY = ON)`, so the base class's seeding `INSERT` is a silent no-op on any database that already holds the row — it neither inserts nor errors — and the value stored on some earlier deployment then wins over the environment variable indefinitely. We reproduced this directly against SQL Server: seeding `3600` and then `60` for the same key reports *"Duplicate key was ignored. (0 rows affected)"* and leaves `3600` in place. A fresh database appears to honor configuration while an upgraded one quietly does not. + +An interim iteration avoided the whole mechanism by not deriving from `Watchdog` at all. We reverted that in favor of keeping the shared base class and using `InitAdditionalParamsAsync`, the one initialization step the base class does make overridable, to `UPDATE` both rows to the configured values and re-assign the properties before the timer is built. Configuration is authoritative on every database, the rows become a readable mirror of what the service is running with rather than an input to it, and no shared file is modified. The distributed lease that stops every replica collecting the same data was kept throughout. ## Consequences @@ -54,6 +58,7 @@ We also decided that every setting lives in configuration and none in `dbo.Param - Diagnostics are enabled and tuned per environment through normal configuration, including an optional run window, and are off by default. - The PHI boundary is enforced in C#, is covered by unit tests, and fails closed rather than emitting an unverified plan. - Collection is single-instance through the existing lease, so the emitted lines are not multiplied by replica count. +- The feature modifies no shared file. It derives from the same `Watchdog` base class as every other watchdog, so it inherits the shared timer, lease orchestration, and any future improvement to them. - Emission costs a log record rather than a charged metric event, so enabling the feature does not add load to the metrics pipeline. ### Adverse effects @@ -61,8 +66,8 @@ We also decided that every setting lives in configuration and none in `dbo.Param - Diagnostics cannot be pulled on demand. Data appears on the collection period, hourly by default, so an incident is served by data that was already being collected rather than by an engineer asking a question and getting an answer straight away. A run window has to be configured in advance. - Settings bind through `IOptions`, so changing them on a running host requires a restart. - Logs are not aggregated for you. Nothing here arrives as a pre-computed time series, so trend questions need a query over the emitted lines rather than a metric chart. -- This watchdog no longer shares the `Watchdog` base class, so it re-implements that class's timer and lease orchestration and will not pick up future improvements to it. That is the cost of keeping configuration out of `dbo.Parameters`, and it should be revisited if the base class itself moves to configuration. -- One shared type changed. `WatchdogLease` was constrained to `T : Watchdog` but used its type argument only for `typeof(T).Name`. The constraint was relaxed so that a component which schedules itself can still elect a single replica. It restricted nothing the class actually used, and every existing caller still satisfies it, but it is a change to a shared file and reviewers should confirm they are comfortable with it. +- This watchdog still writes two rows to `dbo.Parameters`, because the shared base class does so during initialization and we chose to keep that base class rather than fork it. The rows are reconciled to configuration on every start, so they cannot override it, but a reader who inspects the table between a configuration change and the next restart will see values that are briefly out of date. +- The reconciliation is one extra `UPDATE` per process start. It is wrapped in a catch that logs and continues, because it runs inside the base class's initialization, outside the per-tick catch, where an unhandled throw would fault this watchdog's task and cause `WatchdogsBackgroundService` to cancel every other watchdog with it. - Collection depends on Query Store being enabled and in `READ_WRITE` state on the database. Otherwise the job reports why it cannot collect and does nothing. - One piece of pre-existing database state can still suppress collection silently. `dbo.AcquireWatchdogLease` honors watchdog lease include and exclude patterns held in `dbo.Parameters`. A worker excluded by such a row never becomes lease holder, so the feature can be enabled and stay quiet. This applies to every watchdog in the process and is not something this feature sets or reads, but it is the first thing to check on a long-lived database. @@ -70,7 +75,7 @@ We also decided that every setting lives in configuration and none in `dbo.Param - The emitted lines are ordinary log records. There is no handler to bind and this repository prescribes no sink. Whatever a deployment already does with FHIR server logs, it does with these. - The statistics health rows arrive as a JSON array inside one log property rather than as separate columns. A reader has to parse them. That is the trade accepted for batching, and it is affordable because the fields are few and uniform. -- The lease continues to write to its own table. That is runtime coordination rather than configuration, and is not part of what this decision removed. +- The lease continues to write to its own table. That is runtime coordination rather than configuration, and is not part of what this decision changed. ## References diff --git a/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs index 026a5e019e..2cc9ecaea7 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs @@ -24,6 +24,14 @@ public class QueryStoreDiagnosticsConfiguration /// public double PeriodSec { get; set; } = 3600; + /// + /// Gets or sets the lease renewal interval, in seconds, used to elect the single replica that collects each + /// period. Must be positive: it is handed to the lease timer, which rejects a non-positive value. The + /// watchdog base class writes this value into dbo.Parameters, so it is exposed here to keep the + /// stored row settable from an environment variable. + /// + public double LeasePeriodSec { get; set; } = 600; + /// /// Gets or sets the maximum number of slow query plans reported per collection. /// diff --git a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json index c6629db07f..d196e9f626 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json +++ b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json @@ -164,6 +164,7 @@ "QueryStoreDiagnostics": { "Enabled": false, "PeriodSec": 3600, + "LeasePeriodSec": 600, "SlowQueryCount": 10, "MinDurationMilliseconds": 1000, "IncludeQueryPlans": true, diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationOnlyTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationOnlyTests.cs deleted file mode 100644 index e4e486c40f..0000000000 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationOnlyTests.cs +++ /dev/null @@ -1,60 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Linq; -using System.Reflection; -using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; -using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; -using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics.Models; -using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.Test.Utilities; -using Xunit; - -namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics -{ - /// - /// Pins the property the feature is required to have: it is configured entirely through configuration and never - /// reads or writes dbo.Parameters. Both ways that property could be lost are silent — re-deriving from - /// reintroduces the seeding insert without a line of code being written in this - /// feature, and a hand-written statement is only ever exercised against a live database — so neither is caught - /// by the rest of the unit suite. - /// - [Trait(Traits.OwningTeam, OwningTeam.Fhir)] - [Trait(Traits.Category, Categories.Operations)] - public class QueryStoreDiagnosticsConfigurationOnlyTests - { - [Fact] - public void GivenTheWatchdog_WhenItsTypeIsInspected_ThenItDoesNotInheritTheParameterSeedingBaseClass() - { - // Arrange, Act - Type baseType = typeof(QueryStoreDiagnosticsWatchdog).BaseType; - - // Assert - // Watchdog.ExecuteAsync unconditionally awaits a private, non-virtual InitParamsAsync that inserts - // {Name}.PeriodSec and {Name}.LeasePeriodSec into dbo.Parameters and then reads the period back over the - // configured one. There is no hook to suppress it, so not deriving from it is the mechanism by which - // this feature writes nothing, and re-deriving would undo that without touching this feature's code. - Assert.Equal(typeof(object), baseType); - } - - [Fact] - public void GivenEveryStatementTheWatchdogCanIssue_WhenInspected_ThenNoneReadsOrWritesDboParameters() - { - // Arrange - // Every statement this watchdog issues is a const string on the type, so taking all of its string - // literals is a superset of its SQL — the few non-SQL literals caught alongside them cost nothing. - string[] statements = typeof(QueryStoreDiagnosticsWatchdog) - .GetFields(BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public) - .Where(field => field.IsLiteral && field.FieldType == typeof(string)) - .Select(field => (string)field.GetRawConstantValue()) - .ToArray(); - - // Act, Assert - Assert.NotEmpty(statements); - Assert.All(statements, statement => Assert.DoesNotContain("dbo.Parameters", statement, StringComparison.OrdinalIgnoreCase)); - } - } -} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationPrecedenceTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationPrecedenceTests.cs new file mode 100644 index 0000000000..907947906b --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsConfigurationPrecedenceTests.cs @@ -0,0 +1,103 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using System.Reflection; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics +{ + /// + /// Pins the invariant the feature is now required to have: configuration is authoritative over the two + /// dbo.Parameters rows the base class seeds for it. The base class seeds those + /// rows and then reads them back over the configured values, and dbo.Parameters carries + /// IGNORE_DUP_KEY = ON, so on a database that already holds the rows the seeding insert is a silent no-op + /// and a stale row would win over an environment variable. The only thing that stops that is the overridden + /// , which reconciles the rows back to configuration with an + /// UPDATE. Every way that override could be lost is silent — deleting it, or "fixing" the reconciliation + /// into an INSERT that IGNORE_DUP_KEY no-ops, compiles and passes every other unit test, because + /// the reconciliation is only ever exercised against a live database — so these reflection assertions are what + /// catch it. + /// + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public class QueryStoreDiagnosticsConfigurationPrecedenceTests + { + [Fact] + public void GivenTheWatchdog_WhenItsTypeIsInspected_ThenItDerivesFromTheWatchdogBaseClass() + { + // Arrange, Act + Type baseType = typeof(QueryStoreDiagnosticsWatchdog).BaseType; + + // Assert + // Deriving from Watchdog is what supplies the timer and the single-replica lease, and with them the + // seeding of {Name}.PeriodSec and {Name}.LeasePeriodSec into dbo.Parameters. Accepting those rows is + // deliberate; keeping configuration authoritative over them is the job of the InitAdditionalParamsAsync + // override asserted below. + Assert.Equal(typeof(Watchdog), baseType); + } + + [Fact] + public void GivenTheWatchdog_WhenInitAdditionalParamsIsInspected_ThenItIsOverriddenOnTheWatchdog() + { + // Arrange, Act + MethodInfo initAdditionalParams = typeof(QueryStoreDiagnosticsWatchdog).GetMethod( + "InitAdditionalParamsAsync", + BindingFlags.NonPublic | BindingFlags.Instance); + + // Assert + // The base class's InitParamsAsync reads the stored period and lease period back over the configured + // values, then calls this hook and only afterwards builds the timer. Overriding it here is the ONLY + // place configuration is reasserted, so if the declaring type were the base class — the override + // deleted — the stale-row bug would return with no other test failing. + Assert.NotNull(initAdditionalParams); + Assert.Equal(typeof(QueryStoreDiagnosticsWatchdog), initAdditionalParams.DeclaringType); + } + + [Fact] + public void GivenTheReconciliationStatement_WhenInspected_ThenItUpdatesDboParametersAndDoesNotInsert() + { + // Arrange + // The exact statement the override issues, read from the type rather than duplicated, so this fails if the + // real reconciliation ever stops being an UPDATE against dbo.Parameters. + string reconciliationSql = QueryStoreDiagnosticsWatchdog.ReconcileParametersSql; + + // Act, Assert + Assert.Contains("dbo.Parameters", reconciliationSql, StringComparison.OrdinalIgnoreCase); + Assert.Contains("UPDATE", reconciliationSql, StringComparison.OrdinalIgnoreCase); + + // An INSERT here would be silently no-op'd by IGNORE_DUP_KEY on an existing database, leaving the stale + // row in place and reintroducing the bug the override exists to close. + Assert.DoesNotContain("INSERT", reconciliationSql, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void GivenEveryConstStatementTheWatchdogCanIssue_WhenInspected_ThenNoneInsertsIntoDboParameters() + { + // Arrange + // Every statement this watchdog issues is a const string on the type, so taking all of its string + // literals is a superset of its SQL — the few non-SQL literals caught alongside them cost nothing. + string[] statements = typeof(QueryStoreDiagnosticsWatchdog) + .GetFields(BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public) + .Where(field => field.IsLiteral && field.FieldType == typeof(string)) + .Select(field => (string)field.GetRawConstantValue()) + .ToArray(); + + // Act, Assert + // The single UPDATE the reconciliation issues is the whole of what this feature writes to dbo.Parameters; + // no statement inserts into it, which is what keeps IGNORE_DUP_KEY from ever silently ignoring a write + // this feature depended on taking effect. + Assert.NotEmpty(statements); + Assert.All( + statements, + statement => Assert.DoesNotContain("INSERT INTO dbo.Parameters", statement, StringComparison.OrdinalIgnoreCase)); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsLeasePeriodTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsLeasePeriodTests.cs new file mode 100644 index 0000000000..8b8f83044e --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsLeasePeriodTests.cs @@ -0,0 +1,120 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.SqlServer.Features.Storage; +using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnostics; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Watchdogs.QueryStoreDiagnostics +{ + /// + /// Covers the lease renewal interval. Like the collection period it is handed to a timer the base class owns, + /// which rejects a non-positive or non-finite value, and a fault there cancels the token every watchdog shares — + /// so a bad value in this off-by-default feature must degrade to the default rather than fail the host. It is + /// also one of the two values written into dbo.Parameters, which is why it is on the configuration + /// surface at all. + /// + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public class QueryStoreDiagnosticsLeasePeriodTests + { + private const double DefaultLeasePeriodSec = 600; + + [Theory] + [InlineData(0d)] + [InlineData(-1d)] + [InlineData(-600d)] + [InlineData(double.NaN)] + [InlineData(double.PositiveInfinity)] + public void GivenAnUnusableConfiguredLeasePeriod_WhenConstructed_ThenTheDefaultIsUsedAndTheValueIsReported(double configuredLeasePeriodSec) + { + // Arrange + var logger = new CapturingLogger(); + + // Act + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredLeasePeriodSec); + + // Assert + // The lease period must never reach PeriodicTimer in this state: it throws there, which faults this + // watchdog's task and causes WatchdogsBackgroundService to cancel the token every other watchdog shares. + Assert.Equal(DefaultLeasePeriodSec, watchdog.LeasePeriodSec); + + string warning = Assert.Single(logger.WarningMessages); + Assert.Contains(Format(configuredLeasePeriodSec), warning, StringComparison.Ordinal); + Assert.Contains(Format(DefaultLeasePeriodSec), warning, StringComparison.Ordinal); + Assert.Contains(QueryStoreDiagnosticsWatchdog.LeasePeriodSecConfigurationKey, warning, StringComparison.Ordinal); + } + + [Theory] + [InlineData(1d)] + [InlineData(300d)] + [InlineData(600d)] + [InlineData(3600d)] + public void GivenAUsableConfiguredLeasePeriod_WhenConstructed_ThenItIsUsedUnchangedAndNothingIsReported(double configuredLeasePeriodSec) + { + // Arrange + var logger = new CapturingLogger(); + + // Act + QueryStoreDiagnosticsWatchdog watchdog = CreateWatchdog(logger, configuredLeasePeriodSec); + + // Assert + Assert.Equal(configuredLeasePeriodSec, watchdog.LeasePeriodSec); + Assert.Empty(logger.WarningMessages); + } + + private static QueryStoreDiagnosticsWatchdog CreateWatchdog(ILogger logger, double configuredLeasePeriodSec) + { + var configuration = new WatchdogConfiguration(); + configuration.QueryStoreDiagnostics.Enabled = true; + + // Left at its valid class default so that the only value under test is the lease period; a bad period + // would raise its own warning and defeat the Assert.Single checks above. + configuration.QueryStoreDiagnostics.LeasePeriodSec = configuredLeasePeriodSec; + + return new QueryStoreDiagnosticsWatchdog( + Substitute.For(), + logger, + Options.Create(configuration)); + } + + // The logging infrastructure formats message arguments with the invariant culture, so expected values are + // formatted the same way rather than with whatever culture the test host happens to run under. + private static string Format(double value) => value.ToString(CultureInfo.InvariantCulture); + + /// + /// Records what was logged, at what level, with the arguments already substituted, so a test can assert that + /// an operator is told the values they need rather than merely that some warning was raised. + /// + private sealed class CapturingLogger : ILogger + { + private readonly List<(LogLevel Level, string Message)> _entries = new List<(LogLevel Level, string Message)>(); + + internal IReadOnlyList WarningMessages => + _entries.Where(entry => entry.Level == LogLevel.Warning).Select(entry => entry.Message).ToList(); + + public IDisposable BeginScope(TState state) + where TState : notnull + => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + _entries.Add((logLevel, formatter(state, exception))); + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsWatchdog.cs index 558f721b30..eb40ee4c82 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/QueryStoreDiagnostics/QueryStoreDiagnosticsWatchdog.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.Data; -using System.Diagnostics; using System.Linq; using System.Text.Json; using System.Threading; @@ -26,13 +25,13 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs.QueryStoreDiagnosti /// Logs rather than metric events: the payload is not metric-shaped — query text and plan XML are unbounded /// free text and wait categories are high cardinality — and metric events are charged on receipt, which /// docs/arch/adr-2605-metric-emission-rate-limiting.md records as having throttled a shared metric - /// account. Deliberately does not derive from : that base class seeds and then - /// re-reads its period from dbo.Parameters on every start, from a private non-virtual initialization step - /// with no override hook, and this feature is configured exclusively through configuration and must write - /// nothing to the database. The timer and the lease the base class would have supplied are owned directly - /// instead, so the single-collector guarantee is unchanged. + /// account. Derives from for its timer and its single-replica lease, and so accepts + /// the two dbo.Parameters rows the base class seeds for its period and lease period. Configuration stays + /// authoritative over those rows: the base class re-reads them over the configured values during + /// initialization, and is overridden to reconcile them back to + /// configuration before the timer starts, so an environment variable always wins over a stale stored row. /// - internal sealed class QueryStoreDiagnosticsWatchdog + internal sealed class QueryStoreDiagnosticsWatchdog : Watchdog { internal const int MaxFieldLength = 32 * 1024; @@ -42,6 +41,24 @@ internal sealed class QueryStoreDiagnosticsWatchdog /// internal const string PeriodSecConfigurationKey = "FhirServer:Watchdog:QueryStoreDiagnostics:PeriodSec"; + /// + /// The configuration key that sets the lease renewal interval, named in the warning that reports the lease + /// period being unusable so the remedy does not have to be looked up. + /// + internal const string LeasePeriodSecConfigurationKey = "FhirServer:Watchdog:QueryStoreDiagnostics:LeasePeriodSec"; + + /// + /// The statement that reconciles the two dbo.Parameters rows the base class seeds — the period and the + /// lease period — back to the configured values. An UPDATE rather than an INSERT on purpose: + /// the base class's seeding INSERT already guarantees the rows exist, and dbo.Parameters carries + /// IGNORE_DUP_KEY = ON, so a re-INSERT of an existing row is silently ignored and would leave a + /// stale value overriding configuration. Exposed as internal so a unit test can pin that shape without a + /// database, because the reconciliation is otherwise only exercised against a live one. + /// + internal const string ReconcileParametersSql = @" +UPDATE dbo.Parameters SET Number = @PeriodSec WHERE Id = @PeriodSecId +UPDATE dbo.Parameters SET Number = @LeasePeriodSec WHERE Id = @LeasePeriodSecId"; + /// Wait statistics were read for the plan. internal const string WaitStatisticsAvailableStatus = "Available"; @@ -54,6 +71,9 @@ internal sealed class QueryStoreDiagnosticsWatchdog /// The collection interval used when configuration does not supply a usable one. private const double DefaultPeriodSec = 3600; + /// The lease renewal interval used when configuration does not supply a usable one. + private const double DefaultLeasePeriodSec = 600; + /// /// The statistics-health batch size used when configuration does not supply a usable one. Mirrors the class /// default on . @@ -69,26 +89,6 @@ internal sealed class QueryStoreDiagnosticsWatchdog /// private const int MaxStatisticsHealthBatchSize = 64; - /// - /// The lease renewal interval. Ten minutes lets the lease be picked up promptly after a replica dies without - /// expiring in the middle of a collection. It is an internal coordination knob rather than an operator - /// setting — nothing an operator can observe changes with it — so it is deliberately not on the - /// configuration surface. - /// - private const double LeasePeriodSec = 600; - - /// - /// Whether the lease may be handed to another replica to balance watchdogs across a deployment. Matches what - /// every other watchdog asks for. - /// - private const bool AllowLeaseRebalance = true; - - /// - /// The cap on the randomized start-up delay. A period longer than an hour would otherwise leave a restarted - /// host collecting nothing for most of a period before its first tick. - /// - private const double MaxInitialDelaySec = 3600; - /// The shortest lookback window a collection is allowed to use. private const double MinLookbackPeriodSec = 60; @@ -251,8 +251,14 @@ ELSE CONVERT(float, statisticsProperties.modification_counter) / statisticsPrope private readonly QueryStoreDiagnosticsConfiguration _configuration; private readonly ILogger _logger; private readonly ISqlRetryService _sqlRetryService; - private readonly FhirTimer _fhirTimer; - private readonly WatchdogLease _watchdogLease; + + // The validated collection interval and lease renewal interval, resolved from configuration once at + // construction. Held separately from the PeriodSec and LeasePeriodSec properties because the base class + // overwrites those properties with the values it reads back out of dbo.Parameters during initialization; + // InitAdditionalParamsAsync reconciles the table and the properties back to these fields so configuration + // stays authoritative. + private readonly double _effectivePeriodSec; + private readonly double _effectiveLeasePeriodSec; // The run-window state observed on the previous tick, used to log only when the state changes. Null until the // first observation, so that the state a process starts in is always reported once and an operator never has @@ -261,43 +267,59 @@ ELSE CONVERT(float, statisticsProperties.modification_counter) / statisticsPrope // synchronization. private RunWindowState? _lastRunWindowState; - // When the per-tick duration was last reported at information level, so that a short period cannot turn that - // line into noise. Written only from the tick, which FhirTimer runs sequentially. - private DateTime _lastTickReported; - public QueryStoreDiagnosticsWatchdog( ISqlRetryService sqlRetryService, ILogger logger, IOptions watchdogConfiguration) + : base(sqlRetryService, logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _configuration = EnsureArg.IsNotNull(watchdogConfiguration?.Value, nameof(watchdogConfiguration)).QueryStoreDiagnostics; - _fhirTimer = new FhirTimer(_logger); - _watchdogLease = new WatchdogLease(_sqlRetryService, _logger); - // PeriodSec reaches PeriodicTimer through the timer this watchdog now owns (ExecuteAsync -> + // PeriodSec reaches PeriodicTimer through the base class timer (Watchdog.ExecuteAsync -> // FhirTimer.ExecuteAsync -> new PeriodicTimer(TimeSpan.FromSeconds(PeriodSec))), which rejects a - // non-positive period. Owning the timer does not contain that rejection: it would fault this watchdog's - // task, and WatchdogsBackgroundService cancels the token shared by every watchdog as soon as one of - // their tasks completes — so a single mistyped value in an off-by-default diagnostics feature would - // still take the transaction and cleanup watchdogs down with it. A diagnostics feature degrades instead - // of failing the host: keep the class default and name the rejected value. Non-finite values are - // rejected on the same grounds, because TimeSpan.FromSeconds rejects them for the same reason and with - // the same blast radius. + // non-positive period. That rejection is not contained: it faults this watchdog's task, and + // WatchdogsBackgroundService cancels the token shared by every watchdog as soon as one of their tasks + // completes — so a single mistyped value in an off-by-default diagnostics feature would still take the + // transaction and cleanup watchdogs down with it. A diagnostics feature degrades instead of failing the + // host: keep the class default and name the rejected value. Non-finite values are rejected on the same + // grounds, because TimeSpan.FromSeconds rejects them for the same reason and with the same blast radius. if (_configuration.PeriodSec > 0 && double.IsFinite(_configuration.PeriodSec)) { - PeriodSec = _configuration.PeriodSec; + _effectivePeriodSec = _configuration.PeriodSec; } else { - PeriodSec = DefaultPeriodSec; + _effectivePeriodSec = DefaultPeriodSec; _logger.LogWarning( "QueryStoreDiagnosticsWatchdog: configured PeriodSec is {ConfiguredPeriodSec}, which is not a usable collection interval. Falling back to {FallbackPeriodSec} seconds. Configure a positive value in '{PeriodSecConfigurationKey}' to change the interval.", _configuration.PeriodSec, DefaultPeriodSec, PeriodSecConfigurationKey); } + + // The lease renewal interval reaches PeriodicTimer through the base class the same way the period does, + // and a non-positive or non-finite value faults the watchdog with the same shared-token blast radius, so + // it is validated identically and falls back to the class default rather than failing the host. + if (_configuration.LeasePeriodSec > 0 && double.IsFinite(_configuration.LeasePeriodSec)) + { + _effectiveLeasePeriodSec = _configuration.LeasePeriodSec; + } + else + { + _effectiveLeasePeriodSec = DefaultLeasePeriodSec; + _logger.LogWarning( + "QueryStoreDiagnosticsWatchdog: configured LeasePeriodSec is {ConfiguredLeasePeriodSec}, which is not a usable lease renewal interval. Falling back to {FallbackLeasePeriodSec} seconds. Configure a positive value in '{LeasePeriodSecConfigurationKey}' to change the interval.", + _configuration.LeasePeriodSec, + DefaultLeasePeriodSec, + LeasePeriodSecConfigurationKey); + } + + // Assigned here as well as in InitAdditionalParamsAsync so a unit test that never calls ExecuteAsync — + // and so never runs base initialization — still observes the configured values on these properties. + PeriodSec = _effectivePeriodSec; + LeasePeriodSec = _effectiveLeasePeriodSec; } /// @@ -318,86 +340,99 @@ private enum RunWindowState } /// - /// Gets the name this watchdog reports itself under in logs and in the lease it takes. Held as a literal - /// rather than GetType().Name — identical for a sealed class — so that renaming the type surfaces as - /// a deliberate change to a name that appears in operator-facing logs. + /// Gets or sets the interval, in seconds, between collections. Overrides the base class property so the base + /// timer uses it; set from configuration at construction and reconciled back to configuration in + /// after the base class reads it out of dbo.Parameters. /// - public string Name => nameof(QueryStoreDiagnosticsWatchdog); + public override double PeriodSec { get; internal set; } /// - /// Gets the interval, in seconds, between collections. Set once from configuration at construction, because - /// that is the only source for it and does not reload in place. + /// Gets or sets the lease renewal interval, in seconds. Overrides the base class property so the base lease + /// uses it; set from configuration at construction and reconciled back to configuration in + /// after the base class reads it out of dbo.Parameters. /// - public double PeriodSec { get; } + public override double LeasePeriodSec { get; internal set; } /// - /// Exposes RunWorkAsync for unit testing purposes. + /// Gets or sets a value indicating whether the lease may be handed to another replica to balance watchdogs + /// across a deployment. Matches what every other watchdog asks for. /// - /// The cancellation token. - /// A task representing the asynchronous operation. - internal Task RunWorkForTestingAsync(CancellationToken cancellationToken) => RunWorkAsync(cancellationToken); + public override bool AllowRebalance { get; internal set; } = true; /// - /// Runs the collection timer and the lease until the supplied token is cancelled. Called by - /// , which only starts this watchdog when the feature is enabled in - /// configuration. + /// Exposes RunWorkAsync for unit testing purposes. /// /// The cancellation token. /// A task representing the asynchronous operation. - public async Task ExecuteAsync(CancellationToken cancellationToken) - { - _logger.LogDebug("{WatchdogName}.ExecuteAsync: starting...", Name); - - // Reported once per process rather than once per tick. A mistyped window collects nothing and raises - // nothing, which is indistinguishable from a window that has simply not opened yet, so it has to be - // stated at startup — repeating it hourly for the weeks until the window was meant to open would bury - // it. Nothing here touches the database, so it needs no initialization step to hang off. - ReportConfiguredRunWindow(); - - // The timer and the lease run concurrently and neither returns until the token is cancelled. The initial - // delay is randomized up to one period and capped at an hour so that replicas started together do not - // all collect on the same second, and so that a long period does not leave a restarted host silent for - // most of it. - await Task.WhenAll( - _fhirTimer.ExecuteAsync(Name, PeriodSec, OnNextTickAsync, cancellationToken, PeriodSec > MaxInitialDelaySec ? MaxInitialDelaySec : PeriodSec), - _watchdogLease.ExecuteAsync($"{Name}Lease", AllowLeaseRebalance, LeasePeriodSec, cancellationToken)); - - _logger.LogDebug("{WatchdogName}.ExecuteAsync: completed.", Name); - } + internal Task RunWorkForTestingAsync(CancellationToken cancellationToken) => RunWorkAsync(cancellationToken); /// - /// Runs one tick, on the replica that holds the lease. + /// Reconciles the two dbo.Parameters rows the base class seeds — the period and the lease period — back + /// to configuration, and reports the configured run window. Runs after the base class has seeded those rows + /// and read them back over the configured values, and before the base class starts the timer, so the values + /// the timer and the lease run at are the configured ones. + /// + /// This override is the mechanism that keeps configuration authoritative. dbo.Parameters has + /// PRIMARY KEY CLUSTERED (Id) WITH (IGNORE_DUP_KEY = ON), so on a database that already holds these + /// rows the base class's seeding INSERT is a silent no-op — it neither inserts nor errors — and the + /// base class then reads the stale stored value back over the configured one. Without this override an + /// environment variable would be silently overridden by whatever was first stored. The UPDATE forces + /// the stored rows to the configured values, so the table stays an accurate mirror of configuration rather + /// than a misleading stale copy, and the properties are then re-assigned to undo the base class's read-back. + /// An UPDATE is used rather than an INSERT precisely because IGNORE_DUP_KEY would make a + /// re-INSERT a no-op; the base class's seeding INSERT already guarantees the rows exist by the + /// time this hook runs. + /// + /// + /// A failure of the reconciling UPDATE is reported and swallowed. This hook runs inside the base + /// class's initialization, before and outside the per-tick catch, so a throw here would fault this + /// watchdog's task and cause WatchdogsBackgroundService to cancel every other watchdog with it. The + /// property assignments that make configuration authoritative are therefore performed outside the try, so + /// that the functional guarantee does not depend on the cosmetic one. + /// /// - /// The cancellation token. /// A task representing the asynchronous operation. - private async Task OnNextTickAsync(CancellationToken cancellationToken) + protected override async Task InitAdditionalParamsAsync() { - if (!_watchdogLease.IsLeaseHolder) - { - // The lease is what keeps one collection per period rather than one per replica: without this gate - // an eight-instance deployment would issue eight concurrent Query Store scans an hour and emit - // eight copies of every diagnostics line. - _logger.LogDebug("{WatchdogName}.OnNextTickAsync: skipping because this instance does not hold the lease.", Name); - return; - } - - var stopwatch = Stopwatch.StartNew(); - - await RunWorkAsync(cancellationToken); - - // Reports that a tick happened at all, which the collection summary inside RunWorkAsync cannot: a tick - // that returned early — outside the run window, or with Query Store unavailable — logs its reason but - // nothing about the timer still being alive. Throttled to hourly at information level so that a short - // configured period cannot turn it into noise. - if (DateTime.UtcNow - _lastTickReported > TimeSpan.FromHours(1)) + try { - _lastTickReported = DateTime.UtcNow; - _logger.LogInformation("{WatchdogName}.OnNextTickAsync ran in {ElapsedMilliseconds} ms.", Name, stopwatch.ElapsedMilliseconds); + await using var cmd = new SqlCommand(ReconcileParametersSql); + cmd.Parameters.AddWithValue("@PeriodSecId", PeriodSecId); + cmd.Parameters.AddWithValue("@PeriodSec", _effectivePeriodSec); + cmd.Parameters.AddWithValue("@LeasePeriodSecId", LeasePeriodSecId); + cmd.Parameters.AddWithValue("@LeasePeriodSec", _effectiveLeasePeriodSec); + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None, "InitAdditionalParamsAsync failed."); } - else + catch (Exception exception) { - _logger.LogDebug("{WatchdogName}.OnNextTickAsync ran in {ElapsedMilliseconds} ms.", Name, stopwatch.ElapsedMilliseconds); + // Reported and swallowed rather than propagated, and deliberately so. This runs inside + // Watchdog.ExecuteAsync's initialization, which is before and outside FhirTimer's per-tick catch, + // so a throw here faults this watchdog's task — and WatchdogsBackgroundService cancels the token + // shared by EVERY watchdog as soon as one task completes. Letting an off-by-default diagnostics + // feature fail the transaction and cleanup watchdogs over a cosmetic row update is the wrong trade. + // Nothing about the collection depends on the update succeeding: the rows are a mirror of + // configuration, not an input to it, and the assignments below make configuration authoritative in + // this process whether or not the mirror was written. The cost of failure is a stale pair of rows + // that disagree with the running configuration, which is what this warning names. + _logger.LogWarning( + exception, + "{WatchdogName}: could not reconcile the {PeriodSecId} and {LeasePeriodSecId} rows in dbo.Parameters to the configured values. Collection is unaffected and continues to run at the configured period, but those rows may now disagree with configuration and should not be read as the values in use.", + Name, + PeriodSecId, + LeasePeriodSecId); } + + // Undo the base class's read-back of the (now reconciled) stored values, so the properties match + // configuration exactly even if the UPDATE and a concurrent write were to race. + PeriodSec = _effectivePeriodSec; + LeasePeriodSec = _effectiveLeasePeriodSec; + + // Reported once per process rather than once per tick, and from here — after seeding, before the timer + // starts — because that is where the base class initialization lifecycle now provides a single hook. A + // mistyped window collects nothing and raises nothing, which is indistinguishable from a window that has + // simply not opened yet, so it has to be stated at startup; repeating it hourly for the weeks until the + // window was meant to open would bury it. + ReportConfiguredRunWindow(); } /// @@ -555,7 +590,7 @@ private void LogRunWindowState(RunWindowState state, DateTimeOffset? runStartDat } } - private async Task RunWorkAsync(CancellationToken cancellationToken) + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { try { diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs index 410eb14055..4b19944b13 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs @@ -16,11 +16,8 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - // The type argument supplies the lease resource name through typeof(T).Name and nothing else, so it carries no - // constraint. It deliberately does not require Watchdog: a component that schedules itself, as - // QueryStoreDiagnosticsWatchdog does in order to keep its configuration out of dbo.Parameters, still needs to - // elect a single replica. Every existing caller passes a Watchdog and is unaffected. internal class WatchdogLease + where T : Watchdog { private const double TimeoutFactor = 0.25; private readonly object _locker = new(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs index 55df85f98e..a8e08e15ab 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/QueryStoreDiagnosticsWatchdogTests.cs @@ -178,27 +178,34 @@ public async Task GivenConfigurationGateDisabled_WhenRun_ThenNothingIsEmitted() } [Fact] - public async Task GivenTheWatchdogIsStarted_WhenItRuns_ThenItNeitherSeedsNorReadsAnyDboParametersRow() + public async Task GivenExistingStaleParameterRows_WhenTheWatchdogInitialises_ThenTheRowsAreReconciledToConfiguration() { // Arrange - // The feature is configured exclusively through configuration, so starting it must leave dbo.Parameters - // untouched. Only a live database can show that: the seeding insert and the period read this watchdog no - // longer performs both happened at startup, before the first tick, and neither is visible to a test that - // invokes the collection directly. + // The feature now derives from Watchdog, so it accepts the two dbo.Parameters rows the base class + // seeds for its period and lease period. The owner's condition is that those rows stay settable from + // configuration, which only a live database can demonstrate: dbo.Parameters carries IGNORE_DUP_KEY, so + // the base class's seeding INSERT is a silent no-op when the rows already exist and the base class then + // reads the stored value back over the configured one. Without the InitAdditionalParamsAsync override the + // stale row below would win; with it, configuration wins. + const double configuredPeriodSec = 1; + const double configuredLeasePeriodSec = 123; + const double staleValue = 999999; + var logger = new CapturingLogger(); - // A one-second period keeps the randomized start-up delay inside the test's own budget. The lease's first - // acquire attempt is a full lease period away, so no tick of this watchdog reaches a collection here — - // which is the point: what is under test is what running it costs the database before it collects - // anything. - var watchdog = CreateWatchdog(logger, enabled: true, periodSec: 1); + // A one-second period keeps the randomized start-up delay inside the test's own budget. Base + // initialization runs to completion under CancellationToken.None before the timer starts, so the rows are + // reconciled regardless of when the token below trips. + var watchdog = CreateWatchdog(logger, enabled: true, periodSec: configuredPeriodSec, leasePeriodSec: configuredLeasePeriodSec); await using SqlConnection connection = await _fixture.SqlConnectionBuilder.GetSqlConnectionAsync(cancellationToken: CancellationToken.None); await connection.OpenAsync(CancellationToken.None); - // A database this test has run the pre-refactor code against still holds the rows it seeded, and they - // would make the assertion below pass for the wrong reason. + // Start from a known state, then plant stale rows so the assertion proves reconciliation rather than a + // first-time seed. These are exactly the rows a database that ran an earlier configuration would hold. await DeleteWatchdogParametersAsync(connection, CancellationToken.None); + await SeedWatchdogParameterAsync(connection, "QueryStoreDiagnosticsWatchdog.PeriodSec", staleValue, CancellationToken.None); + await SeedWatchdogParameterAsync(connection, "QueryStoreDiagnosticsWatchdog.LeasePeriodSec", staleValue, CancellationToken.None); using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); @@ -209,15 +216,17 @@ public async Task GivenTheWatchdogIsStarted_WhenItRuns_ThenItNeitherSeedsNorRead } catch (OperationCanceledException) { - // Expected whenever the token trips while a randomized start-up delay is still pending, which is the - // usual case. Cancelling between ticks instead returns normally, so neither outcome is asserted on. + // Expected whenever the token trips while a randomized start-up delay or a tick is pending, which is + // the usual case. Cancelling between ticks instead returns normally, so neither outcome is asserted on. } // Assert - // Had the watchdog seeded anything, the rows would be here. Had it read a period or an enablement flag - // from a row it did not seed, it would have thrown InvalidOperationException out of ExecuteAsync rather - // than being cancelled, because no such row exists. - Assert.Equal(0, await CountWatchdogParametersAsync(connection, CancellationToken.None)); + // Both rows exist and hold the configured values, not the stale ones — the UPDATE in the override forced + // the table to mirror configuration. Number is SQL float (IEEE-754 double), so exact equality is correct + // here and no epsilon tolerance is warranted. + Assert.Equal(2, await CountWatchdogParametersAsync(connection, CancellationToken.None)); + Assert.Equal(configuredPeriodSec, await ReadWatchdogParameterAsync(connection, "QueryStoreDiagnosticsWatchdog.PeriodSec", CancellationToken.None)); + Assert.Equal(configuredLeasePeriodSec, await ReadWatchdogParameterAsync(connection, "QueryStoreDiagnosticsWatchdog.LeasePeriodSec", CancellationToken.None)); } private static void AssertStatisticsHealthOrdinals(CapturingLogger logger, string probeTableName) @@ -297,11 +306,12 @@ private static void AssertStatisticsHealthPagination(CapturingLogger logger, int Assert.Equal(expectedRowCount, pages.Sum(page => page.Rows.Count)); } - private QueryStoreDiagnosticsWatchdog CreateWatchdog(ILogger logger, bool enabled, double periodSec = 300) + private QueryStoreDiagnosticsWatchdog CreateWatchdog(ILogger logger, bool enabled, double periodSec = 300, double leasePeriodSec = 600) { var configuration = new WatchdogConfiguration(); configuration.QueryStoreDiagnostics.Enabled = enabled; configuration.QueryStoreDiagnostics.PeriodSec = periodSec; + configuration.QueryStoreDiagnostics.LeasePeriodSec = leasePeriodSec; configuration.QueryStoreDiagnostics.SlowQueryCount = 100; configuration.QueryStoreDiagnostics.MinDurationMilliseconds = 1; configuration.QueryStoreDiagnostics.IncludeQueryPlans = true; @@ -353,13 +363,33 @@ private static async Task DeleteWatchdogParametersAsync(SqlConnection connection private static async Task CountWatchdogParametersAsync(SqlConnection connection, CancellationToken cancellationToken) { - // Matched by prefix rather than by the two names the watchdog used to seed, so a row this feature has no + // Matched by prefix rather than by the two names the watchdog seeds, so a row this feature has no // business creating is caught whatever it is called. await using SqlCommand command = connection.CreateCommand(); command.CommandText = "SELECT COUNT(*) FROM dbo.Parameters WHERE Id LIKE 'QueryStoreDiagnosticsWatchdog%';"; return Convert.ToInt32(await command.ExecuteScalarAsync(cancellationToken)); } + private static async Task SeedWatchdogParameterAsync(SqlConnection connection, string id, double number, CancellationToken cancellationToken) + { + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = "INSERT INTO dbo.Parameters (Id, Number) VALUES (@Id, @Number);"; + command.Parameters.AddWithValue("@Id", id); + command.Parameters.AddWithValue("@Number", number); + await command.ExecuteNonQueryAsync(cancellationToken); + } + + private static async Task ReadWatchdogParameterAsync(SqlConnection connection, string id, CancellationToken cancellationToken) + { + // Number is SQL float, which reads back as a .NET double, so the value is returned unboxed as one and + // compared for exact equality by the caller — no epsilon, because float equality is exactly what the + // reconciliation guarantees. + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT Number FROM dbo.Parameters WHERE Id = @Id;"; + command.Parameters.AddWithValue("@Id", id); + return (double)await command.ExecuteScalarAsync(cancellationToken); + } + private static async Task CreateProbeTableAsync(SqlConnection connection, string tableName, CancellationToken cancellationToken) { // The primary key is named explicitly so that the statistics object it backs has a predictable name to @@ -517,7 +547,7 @@ internal IReadOnlyList StatisticsHealthPages public IDisposable BeginScope(TState state) where TState : notnull - => null; + => NoOpDisposable.Instance; public bool IsEnabled(LogLevel logLevel) => true; @@ -625,6 +655,20 @@ internal StatisticsHealthPage(int pageNumber, int pageCount, int rowCount, List< /// Gets the rows carried on this page, deserialized from the batch property. internal List Rows { get; } } + + /// + /// A scope that does nothing on dispose. The base class's InitParamsAsync opens a timed logging scope + /// through BeginTimedScope, whose ActionTimer dereferences the value BeginScope returns when it is + /// disposed; a real logger returns a non-null scope, so this test double must too. + /// + private sealed class NoOpDisposable : IDisposable + { + internal static readonly NoOpDisposable Instance = new NoOpDisposable(); + + public void Dispose() + { + } + } } } }