Add Query Store performance diagnostics

nuget.config

@@ -1,22 +1,21 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <config>
     <add key="repositoryPath" value="packages" />
   </config>
   <packageSources>
-    <clear />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="Microsoft Health OSS" value="https://microsofthealthoss.pkgs.visualstudio.com/FhirServer/_packaging/Public/nuget/v3/index.json" />
   </packageSources>
   <packageSourceMapping>
     <packageSource key="nuget.org">
       <package pattern="*" />
     </packageSource>
+    <packageSource key="azure-default">
+      <package pattern="*" />
+    </packageSource>
     <packageSource key="Microsoft Health OSS">
       <package pattern="Microsoft.Health.*" />
     </packageSource>
   </packageSourceMapping>
-  <disabledPackageSources>
-    <clear />
-  </disabledPackageSources>
 </configuration>

src/Microsoft.Health.Fhir.Core/Configs/QueryStoreDiagnosticsConfiguration.cs

@@ -0,0 +1,49 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.Core.Configs
+{
+    /// <summary>
+    /// Configuration settings for the Query Store diagnostics watchdog.
+    /// </summary>
+    public class QueryStoreDiagnosticsConfiguration
+    {
+        /// <summary>
+        /// Gets or sets a value indicating whether the Query Store diagnostics watchdog can run.
+        /// The database runtime override must also be enabled.
+        /// </summary>
+        public bool Enabled { get; set; } = false;
+
+        /// <summary>
+        /// Gets or sets the interval, in seconds, between diagnostics collections.
+        /// </summary>
+        public double PeriodSec { get; set; } = 3600;
+
+        /// <summary>
+        /// Gets or sets the maximum number of slow query plans reported per collection.
+        /// </summary>
+        public int SlowQueryCount { get; set; } = 10;
+
+        /// <summary>
+        /// Gets or sets the minimum weighted average plan duration, in milliseconds, to report.
+        /// </summary>
+        public int MinDurationMilliseconds { get; set; } = 1000;
+
+        /// <summary>
+        /// Gets or sets a value indicating whether sanitized query plans are reported.
+        /// </summary>
+        public bool IncludeQueryPlans { get; set; } = true;
+
+        /// <summary>
+        /// Gets or sets a value indicating whether table statistics health is reported.
+        /// </summary>
+        public bool IncludeStatisticsHealth { get; set; } = true;
+
+        /// <summary>
+        /// Gets or sets the maximum number of table statistics rows reported per collection.
+        /// </summary>
+        public int StatisticsHealthCount { get; set; } = 20;
+    }
+}

src/Microsoft.Health.Fhir.Core/Configs/WatchdogConfiguration.cs

@@ -14,5 +14,10 @@ public class WatchdogConfiguration
         /// Gets the expired resource cleanup configuration.
         /// </summary>
         public ExpiredResourceConfiguration ExpiredResource { get; } = new ExpiredResourceConfiguration();
+
+        /// <summary>
+        /// Gets the Query Store diagnostics watchdog configuration.
+        /// </summary>
+        public QueryStoreDiagnosticsConfiguration QueryStoreDiagnostics { get; } = new QueryStoreDiagnosticsConfiguration();
     }
 }

src/Microsoft.Health.Fhir.Core/Features/Metrics/QueryPlanNotification.cs

@@ -0,0 +1,67 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+
+namespace Microsoft.Health.Fhir.Core.Features.Metrics
+{
+    /// <summary>
+    /// Contains a sanitized Query Store execution plan.
+    /// </summary>
+    public class QueryPlanNotification : IMetricsNotification
+    {
+        /// <summary>
+        /// Gets or sets the Query Store query identifier.
+        /// </summary>
+        public long QueryId { get; set; }
+
+        /// <summary>
+        /// Gets or sets the Query Store plan identifier.
+        /// </summary>
+        public long PlanId { get; set; }
+
+        /// <summary>
+        /// Gets or sets the sanitized query plan XML, limited to the diagnostics field-length cap.
+        /// </summary>
+        public string SanitizedQueryPlan { get; set; }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether <see cref="SanitizedQueryPlan"/> was truncated.
+        /// </summary>
+        public bool QueryPlanTruncated { get; set; }
+
+        /// <summary>
+        /// Gets or sets the character length of the raw query plan XML as read from Query Store.
+        /// </summary>
+        public int OriginalQueryPlanLength { get; set; }
+
+        /// <summary>
+        /// Gets or sets the character length of the sanitized query plan XML before truncation.
+        /// Compare this against the field cap to see how much <see cref="SanitizedQueryPlan"/> lost when
+        /// <see cref="QueryPlanTruncated"/> is set. Zero when sanitization did not produce a document.
+        /// </summary>
+        public int SanitizedQueryPlanLength { get; set; }
+
+        /// <summary>
+        /// Gets or sets the outcome of query plan sanitization.
+        /// </summary>
+        public string SanitizationStatus { get; set; }
+
+        /// <summary>
+        /// Gets or sets the timestamp when the notification was created.
+        /// </summary>
+        public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow;
+
+        /// <summary>
+        /// Gets the FHIR operation associated with this notification.
+        /// </summary>
+        public string FhirOperation => "query-store-diagnostics";
+
+        /// <summary>
+        /// Gets the resource type associated with this notification.
+        /// </summary>
+        public string ResourceType => "System";
+    }
+}

src/Microsoft.Health.Fhir.Core/Features/Metrics/SlowQueryNotification.cs

@@ -0,0 +1,130 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+
+namespace Microsoft.Health.Fhir.Core.Features.Metrics
+{
+    /// <summary>
+    /// Contains aggregated Query Store metrics for a slow query plan.
+    /// </summary>
+    public class SlowQueryNotification : IMetricsNotification
+    {
+        /// <summary>
+        /// Gets or sets the Query Store query identifier.
+        /// </summary>
+        public long QueryId { get; set; }
+
+        /// <summary>
+        /// Gets or sets the Query Store plan identifier.
+        /// </summary>
+        public long PlanId { get; set; }
+
+        /// <summary>
+        /// Gets or sets the number of regular completed executions in the reporting interval.
+        /// </summary>
+        public long ExecutionCount { get; set; }
+
+        /// <summary>
+        /// Gets or sets the total execution duration, in milliseconds.
+        /// </summary>
+        public double TotalDurationMilliseconds { get; set; }
+
+        /// <summary>
+        /// Gets or sets the weighted average execution duration, in milliseconds.
+        /// </summary>
+        public double AverageDurationMilliseconds { get; set; }
+
+        /// <summary>
+        /// Gets or sets the maximum execution duration, in milliseconds.
+        /// </summary>
+        public double MaxDurationMilliseconds { get; set; }
+
+        /// <summary>
+        /// Gets or sets the total CPU time, in milliseconds.
+        /// </summary>
+        public double TotalCpuMilliseconds { get; set; }
+
+        /// <summary>
+        /// Gets or sets the weighted average CPU time, in milliseconds.
+        /// </summary>
+        public double AverageCpuMilliseconds { get; set; }
+
+        /// <summary>
+        /// Gets or sets the total logical reads.
+        /// </summary>
+        public double TotalLogicalReads { get; set; }
+
+        /// <summary>
+        /// Gets or sets the weighted average logical reads.
+        /// </summary>
+        public double AverageLogicalReads { get; set; }
+
+        /// <summary>
+        /// Gets or sets the total observed wait time, in milliseconds, when Query Store wait statistics are available.
+        /// </summary>
+        public double? TotalWaitMilliseconds { get; set; }
+
+        /// <summary>
+        /// Gets or sets the average observed wait time per execution, in milliseconds, when Query Store wait statistics are available.
+        /// </summary>
+        public double? AverageWaitMilliseconds { get; set; }
+
+        /// <summary>
+        /// Gets or sets the wait category with the greatest observed wait time, when Query Store wait statistics are available.
+        /// </summary>
+        public string TopWaitCategory { get; set; }
+
+        /// <summary>
+        /// Gets or sets the outcome of wait-statistics collection, so that absent wait fields are self-describing:
+        /// <c>Available</c> when wait statistics were read for this plan, <c>Unavailable</c> when the wait query
+        /// succeeded but returned no row for this plan (typically wait capture is off, or the plan accrued no waits),
+        /// and <c>Failed</c> when the wait query itself threw. <c>Failed</c> means the wait fields are missing because
+        /// collection is broken, not because there was nothing to report.
+        /// </summary>
+        public string WaitStatisticsStatus { get; set; }
+
+        /// <summary>
+        /// Gets or sets the query text, limited to the diagnostics field-length cap.
+        /// </summary>
+        public string QueryText { get; set; }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether <see cref="QueryText"/> was truncated.
+        /// </summary>
+        public bool QueryTextTruncated { get; set; }
+
+        /// <summary>
+        /// Gets or sets the character length of the query text before truncation, so the amount lost is
+        /// visible when <see cref="QueryTextTruncated"/> is set.
+        /// </summary>
+        public int QueryTextLength { get; set; }
+
+        /// <summary>
+        /// Gets or sets the start of the Query Store reporting interval.
+        /// </summary>
+        public DateTimeOffset IntervalStart { get; set; }
+
+        /// <summary>
+        /// Gets or sets the end of the Query Store reporting interval.
+        /// </summary>
+        public DateTimeOffset IntervalEnd { get; set; }
+
+        /// <summary>
+        /// Gets or sets the timestamp when the notification was created.
+        /// </summary>
+        public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow;
+
+        /// <summary>
+        /// Gets the FHIR operation associated with this notification.
+        /// </summary>
+        public string FhirOperation => "query-store-diagnostics";
+
+        /// <summary>
+        /// Gets the resource type associated with this notification.
+        /// </summary>
+        public string ResourceType => "System";
+    }
+}

src/Microsoft.Health.Fhir.Core/Features/Metrics/StatisticsHealthNotification.cs

@@ -0,0 +1,90 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+
+namespace Microsoft.Health.Fhir.Core.Features.Metrics
+{
+    /// <summary>
+    /// Contains table statistics health information.
+    /// </summary>
+    public class StatisticsHealthNotification : IMetricsNotification
+    {
+        /// <summary>
+        /// Gets or sets the schema that owns the table.
+        /// </summary>
+        public string SchemaName { get; set; }
+
+        /// <summary>
+        /// Gets or sets the table name.
+        /// </summary>
+        public string TableName { get; set; }
+
+        /// <summary>
+        /// Gets or sets the statistics object name.
+        /// </summary>
+        public string StatisticsName { get; set; }
+
+        /// <summary>
+        /// Gets or sets the timestamp when the statistics were last updated.
+        /// </summary>
+        public DateTimeOffset? LastUpdated { get; set; }
+
+        /// <summary>
+        /// Gets or sets the number of rows represented by the statistics.
+        /// </summary>
+        public long? Rows { get; set; }
+
+        /// <summary>
+        /// Gets or sets the number of rows sampled to build the statistics.
+        /// </summary>
+        public long? RowsSampled { get; set; }
+
+        /// <summary>
+        /// Gets or sets the number of modifications since the statistics were last updated.
+        /// </summary>
+        public long? ModificationCounter { get; set; }
+
+        /// <summary>
+        /// Gets or sets the percentage of represented rows modified since the statistics were last updated.
+        /// </summary>
+        public double? ModificationPercent { get; set; }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether SQL Server automatically created the statistics.
+        /// </summary>
+        public bool IsAutoCreated { get; set; }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether a user created the statistics.
+        /// </summary>
+        public bool IsUserCreated { get; set; }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether the statistics are associated with an index.
+        /// </summary>
+        public bool IsFromIndex { get; set; }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether the statistics use a filter.
+        /// </summary>
+        public bool HasFilter { get; set; }
+
+        /// <summary>
+        /// Gets or sets the timestamp when the notification was created.
+        /// </summary>
+        public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow;
+
+        /// <summary>
+        /// Gets the FHIR operation associated with this notification.
+        /// </summary>
+        public string FhirOperation => "query-store-diagnostics";
+
+        /// <summary>
+        /// Gets the resource type associated with this notification.
+        /// </summary>
+        public string ResourceType => "System";
+    }
+}

src/Microsoft.Health.Fhir.Shared.Web/appsettings.json

@@ -159,6 +159,15 @@
         "Watchdog": {
             "ExpiredResource": {
                 "Enabled": false
+            },
+            "QueryStoreDiagnostics": {
+                "Enabled": false,
+                "PeriodSec": 3600,
+                "SlowQueryCount": 10,
+                "MinDurationMilliseconds": 1000,
+                "IncludeQueryPlans": true,
+                "IncludeStatisticsHealth": true,
+                "StatisticsHealthCount": 20
             }
         }
     },