diff --git a/.gitignore b/.gitignore index 1f14fb8c87..aafe08d062 100644 --- a/.gitignore +++ b/.gitignore @@ -97,6 +97,9 @@ ipch/ # Visual Studio Code C# Dev Kit cache files *.lscache +# VS Code personal workspace settings (launch.json, mcp.json, and tasks.json are tracked intentionally) +.vscode/settings.json + # TFS 2012 Local Workspace $tf/ diff --git a/Directory.Packages.props b/Directory.Packages.props index cb9e062442..3a8508b0a8 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,7 +1,8 @@ - 11.0.135 + + 1.0.0-fix-vector-schema-model-20260720-144944-preview 5.11.4 5.11.0 10.0.10 @@ -40,6 +41,7 @@ + @@ -131,6 +133,7 @@ + diff --git a/THIRDPARTYNOTICES.md b/THIRDPARTYNOTICES.md index ac793ce6fa..9fc5f79e6d 100644 --- a/THIRDPARTYNOTICES.md +++ b/THIRDPARTYNOTICES.md @@ -805,6 +805,23 @@ This file is based on or incorporates material from the projects listed below (T > (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS > SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +## PdfPig 0.1.15 +* Component Source: https://github.com/UglyToad/PdfPig +* Component Copyright and License: Apache License 2.0 + > This product is derived from software developed at + > The Apache Software Foundation (http://www.apache.org/). + > + > Based on source code originally developed in the PDFBox and + > FontBox projects. + > + > Copyright (c) 2002-2007, www.pdfbox.org + > + > Includes the Adobe Glyph List + > Copyright 1997, 1998, 2002, 2007, 2010 Adobe Systems Incorporated. + > + > Includes the Zapf Dingbats Glyph List + > Copyright 2002, 2010 Adobe Systems Incorporated. + ## prometheus-net.AspNetCore 4.1.1 * Component Source: https://github.com/prometheus-net/prometheus-net * Component Copyright and License: diff --git a/build/jobs/scripts/Provision-AcaDeploy.ps1 b/build/jobs/scripts/Provision-AcaDeploy.ps1 index 24c6b95750..6896a7ba26 100644 --- a/build/jobs/scripts/Provision-AcaDeploy.ps1 +++ b/build/jobs/scripts/Provision-AcaDeploy.ps1 @@ -26,8 +26,10 @@ param( [Parameter(Mandatory = $true)] [string] $TenantIdGuid, [Parameter(Mandatory = $false)] [string] $SqlServerName = '', + [Parameter(Mandatory = $false)] [string] $SqlDatabaseName = '', [Parameter(Mandatory = $false)] [string] $SqlElasticPoolName = '', [Parameter(Mandatory = $false)] [string] $SchemaAutomaticUpdatesEnabled = 'auto', + [Parameter(Mandatory = $false)] [ValidateSet('true', 'false')] [string] $DeleteAllDataOnStartup = 'false', [Parameter(Mandatory = $false)] [string] $ReindexEnabled = 'true', # ACA scaling/sizing (sourced from ci-variables.yml / pr-variables.yml) @@ -50,6 +52,7 @@ $ErrorActionPreference = 'Stop' # Bool-ish string args from the YAML wrapper get parsed here so that 'false' actually means false. $reindexEnabledBool = $ReindexEnabled -eq 'true' +$deleteAllDataOnStartupBool = $DeleteAllDataOnStartup -eq 'true' Add-Type -AssemblyName System.Web @@ -142,7 +145,7 @@ $resourceGroupName = $ResourceGroup # --- Data-store-specific pre-deploy setup --- if ($DataStore -eq 'sql') { $sqlServerName = $SqlServerName.ToLowerInvariant() - $sqlDatabaseName = "FHIR$Version" + $sqlDatabaseName = if ([string]::IsNullOrWhiteSpace($SqlDatabaseName)) { "FHIR$Version" } else { $SqlDatabaseName } $sqlElasticPoolName = $SqlElasticPoolName $existingDb = Get-AzSqlDatabase -ResourceGroupName $resourceGroupName -ServerName $sqlServerName -DatabaseName $sqlDatabaseName -ErrorAction SilentlyContinue if ($null -eq $existingDb) { @@ -274,7 +277,9 @@ $templateParameters = @{ if ($DataStore -eq 'sql') { $templateParameters["sqlServerName"] = $sqlServerName + $templateParameters["sqlDatabaseName"] = $sqlDatabaseName $templateParameters["sqlSchemaAutomaticUpdatesEnabled"] = $SchemaAutomaticUpdatesEnabled + $templateParameters["deleteAllDataOnStartup"] = $deleteAllDataOnStartupBool } else { $templateParameters["cosmosDbAccountName"] = $cosmosDbAccountName } diff --git a/docs/SemanticSearch.md b/docs/SemanticSearch.md new file mode 100644 index 0000000000..c014108dc2 --- /dev/null +++ b/docs/SemanticSearch.md @@ -0,0 +1,621 @@ +# Semantic Search (SQL) + +Internal engineering reference for the SQL semantic (vector) search capability introduced on the +`feature/sql-semantic-search` branch. This document describes exactly what the branch adds to the +repository, the formal FHIR contracts it defines (the custom SearchParameter, its configuration +extension, the evidence extension, and the Patient operation), the SQL schema, the runtime +configuration, and how the write, query, reindex, and refresh flows work. + +This is a reference for the team, not customer-facing documentation. The narrative rationale and the +alternatives that were considered live in the ADR ([docs/arch/adr-2608-sql-semantic-search.md](arch/adr-2608-sql-semantic-search.md)) +and in the design spec that accompanies it. + +- Status: experimental, SQL Server only, disabled by default. +- Storage contract: SQL Server 2025 native `vector(1536)`, cosine distance, exact nearest-chunk ranking. +- Scope of retrieval: the FHIR server ranks and returns evidence only. Intent detection and answer + generation are out of scope and remain the caller's responsibility. + +--- + +## 1. At a glance + +The branch adds a metadata-driven vector search feature that is expressed entirely through FHIR +SearchParameter metadata, so no resource type or parameter identity is hard-coded. + +- A new SearchParameter of type `special` with code `semantic-text`, carrying a Microsoft + `vector-search-config` extension, marks a resource element as vectorizable and configures how it is + chunked, embedded, and queried. +- On write, the server extracts the configured text (directly, or from a referenced local `Binary`), + chunks it, calls an external embedding endpoint, and stores one vector row per chunk. Embedding is + synchronous and the vectors are committed in the same transaction as the resource. +- On query, `?semantic-text=` combines with ordinary FHIR filters. The structured filters and the + Patient compartment bound the candidate set, then SQL ranks that set by cosine distance and returns + each matched resource once with a relevance score and evidence (the exact matched chunk plus its + provenance). +- A Patient-scoped operation, `POST [base]/Patient/{id}/$semantic-search`, ranks across every + vector-eligible resource type in the patient compartment in one call. +- `$reindex` backfills vectors for existing resources, and a durable job re-embeds owners when a + referenced `Binary` changes or is deleted. + +--- + +## 2. Concepts and terminology + +- Vector SearchParameter: an ordinary FHIR `SearchParameter` resource of type `special` whose + `vector-search-config` extension makes its `base`/`expression` a vectorization target. Discovered from + the live registry; never hard-coded. +- Chunk: a bounded slice of source text (token window with overlap). One embedding and one + `dbo.VectorSearchParam` row is produced per chunk. +- Embedding model registry: a SQL table (`dbo.EmbeddingModel`) that stamps every vector with the model + name, version, dimension, and distance metric that produced it. +- Evidence: the per-match record of the exact chunk that ranked, its score and rank, the vector + SearchParameter canonical, the source resource and element path, and (for chained queries) the witness + resource whose vector produced the match. +- Source strategy: whether the configured expression yields the text directly (`DirectText`) or a + reference to a local `Binary` whose content is the text (`LocalBinaryReference`). + +--- + +## 3. Client-facing surfaces + +### 3.1 The `semantic-text` SearchParameter (type `special`) + +Semantic search is modeled as a FHIR `SearchParameter` of type `special` (the same mechanism the spec +uses for `near` and `_text`). Its code is `semantic-text`. It is not hard-coded: an operator registers a +`SearchParameter` resource whose `base` and `expression` point at the element to vectorize and whose +`vector-search-config` extension configures behavior. The parameter appears in the `CapabilityStatement` +once active, supported, and searchable. + +Ordinary resource search, combining structured filters and the semantic predicate: + +```http +GET [base]/DocumentReference?patient=Patient/123&date=ge2026-06-05&semantic-text=trouble%20breathing +``` + +Long queries can be sent as `POST [base]/DocumentReference/_search` with a form body. The response is a +normal `searchset` Bundle. Each match carries a relevance score in `Bundle.entry.search.score` and a +`semantic-search-evidence` extension (section 3.4). + +Eligibility rules enforced by `VectorSearchParameterResolver`: a definition is used for indexing and +query only when it is type `special`, carries valid `vector-search-config` metadata, is active, has a +`base` and `expression`, and is both supported and searchable. A newly posted definition that is only +`Supported` is admitted for first-activation backfill but not for query until it is searchable. + +Parsing: `SearchParameterExpressionParser` maps a `special` parameter that carries `vector-search-config` +to an immutable `VectorSearchExpression`. Query text is preserved verbatim (including commas) and is +excluded from the plan-shape `ToString()`. Modifiers are rejected. Absent semantic services, an +unregistered canonical, or an inactive/unsupported/non-searchable definition produce +`SearchParameterNotSupportedException`. + +### 3.2 The `vector-search-config` SearchParameter extension + +Canonical URL: `http://microsoft.com/fhir/StructureDefinition/vector-search-config`. +Model: `Microsoft.Health.Fhir.Core.Models.VectorSearchParameterConfig`. Parsed onto +`SearchParameterInfo.VectorConfig` by `SearchParameterWrapper`. + +| Nested extension | Type | Meaning | Default | +|---|---|---|---| +| `extractionPolicy` | code | How expression values become source text: `FirstValue`, `Concatenate`, or `PerValueRow`. | `Concatenate` | +| `sourceStrategy` | code | `DirectText` (value is the text) or `LocalBinaryReference` (value is a `Binary` reference whose content is the text). | `DirectText` | +| `maxInputTokens` | integer | Upper bound on source tokens accepted from this parameter. | `8000` | +| `minimumScore` | decimal | Minimum normalized score (0..1) for a chunk to be eligible. Acts as a maximum cosine-distance predicate. | `0` | +| `chunkSizeTokens` | integer (optional) | Per-parameter chunk size. Falls back to the server default when omitted. | server default | +| `chunkOverlapTokens` | integer (optional) | Per-parameter chunk overlap. Falls back to the server default when omitted. | server default | +| `distanceMetric` | string (optional) | Per-parameter metric. Only `cosine` is supported today. | `cosine` | + +`extractionPolicy` values are defined by `VectorTextExtractionPolicy`; `sourceStrategy` values by +`VectorTextSourceStrategy`. + +### 3.3 The `$semantic-search` Patient operation + +Defined by `Microsoft.Health.Fhir.Core/Data/OperationDefinition/semantic-search.json` and served by +`SemanticSearchController`. Route: `POST [base]/Patient/{id}/$semantic-search` +(`KnownRoutes.SemanticSearchPatientById`). Instance-level on `Patient`, `affectsState = false`, +`experimental = true`. + +Input parameters (`Parameters` body): + +| Name | Card. | Type | Meaning | +|---|---|---|---| +| `query` | 1..1 | string | Natural-language text to rank the patient compartment by. | +| `type` | 0..* | code | Optional resource types to include. Repeating forms a union. | +| `count` | 0..1 | integer | Maximum globally ranked results. Validated against `Query.MaxCount`. | + +Output: + +| Name | Card. | Type | Meaning | +|---|---|---|---| +| `return` | 1..1 | Bundle | A `searchset` Bundle of globally ranked resources with semantic evidence. | + +Request example: + +```http +POST [base]/Patient/123/$semantic-search +Content-Type: application/fhir+json + +{ + "resourceType": "Parameters", + "parameter": [ + { "name": "query", "valueString": "trouble breathing overnight" }, + { "name": "count", "valueInteger": 10 }, + { "name": "type", "valueCode": "DocumentReference" } + ] +} +``` + +Candidate types are the intersection of the FHIR Patient-compartment resource types, the resource types +that have active vector SearchParameters, and any requested `type` values. Requesting an ineligible type +fails explicitly. Candidate selection uses one `ISearchService.SearchCompartmentAsync` query; no resource +type is hard-coded in the controller or handler. `SemanticSearchController` dispatches a +`SemanticSearchRequest` through the mediator; `SemanticSearchHandler` performs compartment selection, +per-type vector ranking, global ranking, evidence authorization, and Bundle construction. + +### 3.4 The `semantic-search-evidence` extension + +Canonical URL: `http://microsoft.com/fhir/StructureDefinition/semantic-search-evidence`. +Written on `Bundle.entry.search` by `BundleFactory`. Model: +`Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch.SemanticSearchEvidence`. + +| Nested extension | Type | Meaning | +|---|---|---| +| `text` | string | The exact matched chunk text. | +| `chunkOrdinal` | integer | Zero-based ordinal of the chunk within the indexed text. | +| `score` | decimal | Normalized chunk relevance (0..1, higher is more relevant). | +| `rank` | positiveInt | One-based rank across all evidence on the current response page, by score descending. | +| `searchParameter` | uri | Canonical of the vector SearchParameter that selected the text. | +| `source` | Reference | Resource that contains the source text (for example a `Binary`). | +| `sourcePath` | string | Element path of the source text (for example `Binary.data` or `Binary.data#page=2`). | +| `witness` | Reference (optional) | The related vector-owning resource that produced a chained match. | + +Multiple matching chunks do not duplicate the Bundle entry: each matched resource appears once, and up to +`Query.EvidenceCount` evidence items are attached. Rank is assigned after resource pagination by +`SemanticSearchEvidenceRanker`, so it is stable within a page and restarts across pages. + +Serialization note: ordinary search Bundles normally use an optimized raw serializer that emits only +`entry.search.mode`. `FhirJsonOutputFormatter` falls back to the standard Firely serializer when any entry +has `Search.Score` or `Search.Extension`, so score and evidence survive without hard-coding semantic +fields. + +### 3.5 Relevance score, ranking, and sorting + +`VECTOR_DISTANCE('cosine', a, b)` returns 0 (identical) to 2 (opposite). The server normalizes it to a +0..1 relevance score as `1 - distance / 2`, clamped to `[0,1]`, and writes it to +`Bundle.entry.search.score` and to each evidence `score`. + +Ordering contract: + +- No `_sort` and `_sort=_score` both order by semantic distance ascending (best score first), with + resource type and surrogate id as deterministic tie-breakers. +- An explicit ordinary FHIR `_sort` (for example `-_lastUpdated`) overrides relevance ordering while the + score and evidence remain on each result. +- `_score` is a synthetic sort token accepted only when the completed query contains a vector predicate; + `-_score`, and `_score` without a vector predicate, are rejected. +- Semantic continuation tokens carry distance, resource type id, and surrogate id, so paging stays stable + under default and explicit score order. + +### 3.6 Semantic chaining (one level) + +The server supports exactly one chain hop, forward or reverse, when the vector SearchParameter uses a +direct-text or a linked-Binary source: + +- Reverse: `GET [base]/Patient?_has:Observation:subject:semantic-text=` ranks each `Patient` by the + best matching witness `Observation`. +- Forward: `GET [base]/Observation?subject:Patient.semantic-text=` ranks each `Observation` by its + referenced `Patient` target's vectors. + +Each root resource is returned once (a `CROSS APPLY TOP (1)` selects the best witness), and the evidence +carries the witness reference separately from the source reference. Multi-hop chains and, for chained +queries, linked-Binary reverse chains that require intermediate-witness authorization are rejected before +the embedding call. Multi-hop assembly beyond one level is the caller's responsibility using ordinary FHIR +(`_include`, `_revinclude`, `Encounter/$everything`). + +--- + +## 4. Configuration + +All settings live under `FhirServer:CoreFeatures:VectorSearch` +(`Microsoft.Health.Fhir.Core.Configs.VectorSearchConfiguration`) and are validated at startup when the +feature is enabled, so a misconfigured deployment fails fast. The feature is off by default. + +```jsonc +{ + "FhirServer": { + "CoreFeatures": { + "VectorSearch": { + "Enabled": true, + "Embedding": { + "Endpoint": "https://.cognitiveservices.azure.com", + "DeploymentName": "text-embedding-3-small", + "ModelName": "text-embedding-3-small", + "ModelVersion": "1", + "Dimensions": 1536 + }, + "Indexing": { + "Mode": "Synchronous", + "ChunkSizeTokens": 800, + "ChunkOverlapTokens": 100, + "Pdf": { + "MaximumFileSizeBytes": 10485760, + "MaximumPageCount": 200, + "MaximumExtractedCharacters": 500000, + "ExtractionTimeout": "00:00:30" + } + }, + "Query": { + "DefaultCount": 10, + "MaxCount": 50, + "CandidateCount": 100, + "EvidenceCount": 3, + "DistanceMetric": "cosine" + } + } + } + } +} +``` + +| Setting | Default | Validation when enabled | +|---|---|---| +| `Enabled` | `false` | When false, vector SearchParameters are inert and no semantic services are registered. | +| `Embedding.Endpoint` | none | Required, must be an absolute HTTPS URI. | +| `Embedding.DeploymentName` | none | Required, non-empty. | +| `Embedding.ModelName` | `text-embedding-3-small` | Required, non-empty. | +| `Embedding.ModelVersion` | none | Required, non-empty. | +| `Embedding.Dimensions` | `1536` | Must equal `1536` (the SQL vector width). | +| `Indexing.Mode` | `Synchronous` | Only `Synchronous` is supported. | +| `Indexing.ChunkSizeTokens` | `800` | Must be greater than zero. | +| `Indexing.ChunkOverlapTokens` | `100` | Non-negative and strictly less than the chunk size. | +| `Indexing.Pdf.MaximumFileSizeBytes` | `10485760` (10 MiB) | Greater than zero. | +| `Indexing.Pdf.MaximumPageCount` | `200` | Greater than zero. | +| `Indexing.Pdf.MaximumExtractedCharacters` | `500000` | Greater than zero. | +| `Indexing.Pdf.ExtractionTimeout` | `00:00:30` | Greater than zero. | +| `Query.DefaultCount` | `10` | Greater than zero. | +| `Query.MaxCount` | `50` | Greater than or equal to `DefaultCount`. | +| `Query.CandidateCount` | `100` | Greater than or equal to `MaxCount`. | +| `Query.EvidenceCount` | `3` | Greater than zero. | +| `Query.DistanceMetric` | `cosine` | Must equal `cosine`. | + +Constants `VectorSearchConfiguration.SupportedDimensions = 1536` and +`SupportedDistanceMetric = "cosine"` encode the current storage contract. + +--- + +## 5. SQL schema + +The feature adds vector storage keyed the same way as the existing search-parameter tables, so ordinary +structured filters continue to run through the existing tables and only the embedding is new. Schema +versions 117 through 119 are introduced (`SchemaVersionConstants.Max = 119`). + +### 5.1 Tables and types + +`dbo.VectorSearchParam` (one row per chunk): + +```sql +CREATE TABLE dbo.VectorSearchParam +( + ResourceTypeId smallint NOT NULL, + ResourceSurrogateId bigint NOT NULL, + SearchParamId smallint NOT NULL, + ChunkOrdinal smallint NOT NULL, -- default 0 + EmbeddingModelId smallint NOT NULL, + ChunkText nvarchar(max) NOT NULL, -- exact chunk returned as evidence + SourceTextHash binary(32) NOT NULL, -- chunk content hash (provenance; reserved for a future reuse optimization) + SourceResourceTypeId smallint NULL, -- provenance for referenced-source text + SourceResourceId varchar(64) NULL, + SourceResourceVersion varchar(64) NULL, + SourcePath nvarchar(512) NULL, + Embedding vector(1536) NOT NULL, + CONSTRAINT PKC_VectorSearchParam PRIMARY KEY CLUSTERED + (ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal) +); + +-- Reverse lookup: given a changed source (for example a Binary), find owners to refresh. +CREATE NONCLUSTERED INDEX IX_VectorSearchParam_SourceResource +ON dbo.VectorSearchParam (SourceResourceTypeId, SourceResourceId) +INCLUDE (ResourceTypeId, ResourceSurrogateId) +WHERE SourceResourceTypeId IS NOT NULL AND SourceResourceId IS NOT NULL; +``` + +`dbo.EmbeddingModel` (model registry, keyed by name and version): + +```sql +CREATE TABLE dbo.EmbeddingModel +( + EmbeddingModelId smallint IDENTITY(1,1) NOT NULL, + ModelName varchar(128) NOT NULL, + ModelVersion varchar(64) NOT NULL, + Dimension int NOT NULL, + DistanceMetric varchar(16) NOT NULL, -- default 'cosine' + CreatedAt datetime2(7) NOT NULL, -- default sysutcdatetime() + CONSTRAINT PKC_EmbeddingModel PRIMARY KEY CLUSTERED (EmbeddingModelId), + CONSTRAINT U_EmbeddingModel_Name_Version UNIQUE (ModelName, ModelVersion) +); +``` + +`dbo.VectorSearchParamList` is the table-valued parameter used to send chunk rows to the merge +procedures. Its `Embedding` column is passed as `nvarchar(max)` JSON and cast to `vector(1536)` in SQL. + +The chunking policy is not stored on `EmbeddingModel`. It comes from configuration and, optionally, from +the per-parameter `vector-search-config` extension. A chunk-size change is treated like a model change and +triggers re-embedding of affected rows on the next write or reindex. + +The embedding endpoint and deployment name are deployment configuration (`VectorSearch.Embedding`), not +durable model metadata: `EmbeddingModel` stores only the model name, version, dimension, and distance +metric that produced each vector. + +### 5.2 Stored procedures + +- `MergeResources`, `MergeResourcesAndSearchParams`: extended to accept and persist the vector TVP. +- `UpdateResourceSearchParamsWithVectors` (v118): wraps ordinary and vector index updates in one + transaction for reindex, replacing vectors only for the current type, surrogate, resource id, and + version. +- `MergeResourcesWithVectorSearchSourceRefresh`, + `MergeResourcesAndSearchParamsWithVectorSearchSourceRefresh`, + `MergeResourcesDeleteResourceWithVectorSearchSourceRefresh` (v119): enqueue linked-source refresh work + atomically at the owning procedure's commit boundary on write and hard delete. +- `EnqueueVectorSearchSourceRefreshJobs`, `GetVectorSearchSourceDependencies` (v119): enqueue refresh jobs + and resolve the owners that depend on a changed source. + +### 5.3 Schema version milestones + +- V117: `dbo.VectorSearchParam`, `dbo.EmbeddingModel`, `dbo.VectorSearchParamList`, and the merge-path + wiring that persists vectors on write. +- V118 (`VectorSearchReindexVersion`): `UpdateResourceSearchParamsWithVectors` for vector-aware + `$reindex`. +- V119 (`VectorSearchSourceRefreshVersion`): durable linked-source refresh (reindex queue type, job type, + reverse-dependency lookup, and the refresh worker). + +--- + +## 6. How it works + +### 6.1 Write and index flow + +Triggered synchronously inside the resource write in `SqlServerFhirDataStore` when semantic search is +registered and enabled. + +1. `TypedElementSearchIndexer.Extract` produces search-index entries. Vector parameters (type `special` + with `VectorConfig`) are filtered out of the ordinary search-value buckets at the two SQL persistence + sites (`MergeSearchParameterRowGenerator`, `ResourceWriteClaimListRowGenerator`) so they do not flow + into the token/string tables, but the entries remain visible to the vector indexer. +2. `VectorTextSourceResolver` resolves source text for each vector parameter. For `DirectText` it uses the + extracted value; for `LocalBinaryReference` it resolves the referenced `Binary` and decodes its content + through an `IBinaryContentExtractor` selected by MIME type (`PlainTextBinaryContentExtractor` for + `text/plain`, `PdfBinaryContentExtractor` for `application/pdf`). PDF extraction is bounded by the + `Pdf` limits and emits one segment per page with `page=N` provenance. Unsupported or non-text content + is skipped without failing the write. +3. Text is normalized (whitespace, line endings, control characters) so an unchanged note produces a + stable `SourceTextHash`. +4. `TextChunker` splits the text into ordered, overlapping chunks using the active chunk size and overlap. +5. Every chunk is embedded by `IEmbeddingClient` (`AzureFoundryEmbeddingClient`) on each write. The + indexer does not currently skip unchanged chunks: it re-extracts, re-chunks, and re-embeds all chunks + and replaces the resource's vectors. Every produced vector is stamped with the `EmbeddingModelId` + resolved by `SqlEmbeddingModelRegistry` from `(ModelName, ModelVersion)`, and each chunk stores a + `SourceTextHash` so a future reuse optimization can skip re-embedding only when both the hash and the + active `EmbeddingModelId` match (a hash match alone must not reuse a vector produced by a different + model). +6. `VectorSearchIndexer.IndexAsync` produces the vector index entries; `VectorSearchParamListRowGenerator` + turns them into TVP rows; the merge procedures replace the resource's vectors in the same transaction + as the resource. Only the current version is vectorized. + +`ResourceWrapper.VectorSearchIndicesUpdated` distinguishes an intentional empty vector result (delete +stale rows) from semantic indexing being disabled (preserve existing vectors). + +### 6.2 Query and rank flow + +1. `SearchParameterExpressionParser` recognizes `semantic-text` and builds a `VectorSearchExpression`, + preserving any one-level chain relationship around the vector leaf. +2. `VectorSearchQueryProcessor` prepares the query once, before retries and query-cache races: it finds + exactly one vector expression, embeds the query text through `IEmbeddingClient`, validates the fixed + 1536-dimension contract, resolves the SQL-local `EmbeddingModelId`, and returns an immutable + `PreparedVectorSearchQuery` (including `PreparedVectorSearchChainLink`s for chained queries). Duplicate + vector expressions are rejected before any external call. +3. `RemoveVectorSearchRewriter` removes the vector leaf from the structured predicate tree while leaving + the structured filters intact, so the SQL query filters candidates first. +4. `SqlQueryGenerator` emits SQL that applies the structured filters and Patient compartment, then ranks + the bounded candidate set (`Query.CandidateCount`) by `VECTOR_DISTANCE('cosine', ...)`. Candidates are + deduplicated by resource using each resource's highest-scoring chunk (a `CROSS APPLY TOP (1)` per + owner), so each resource is returned once, and the resource limit (`count`) is applied to that + deduplicated set. Additional chunks for a returned resource may still be attached as evidence (up to + `Query.EvidenceCount`). `SqlVectorStore`, `SqlDocumentReferenceSemanticSearch`, and + `SqlVectorResourceReader` execute and read results; `SqlVectorFormatter` handles the vector parameter + encoding. +5. `SearchResultEntry` carries the score and the winning evidence. `SemanticSearchEvidenceRanker` assigns + page ranks. `SemanticSearchEvidenceFilter` authorizes evidence sources (section 7). +6. `BundleFactory` writes `Bundle.entry.search.score` and the `semantic-search-evidence` extension. + +The Patient operation follows the same ranking and evidence path per candidate type, then ranks globally +across types in `SemanticSearchHandler`. + +### 6.3 Reindex backfill + +`$reindex` populates vectors for resources written before the feature or before a parameter was +activated. `ReindexProcessingJob` invokes `IVectorSearchIndexer` once per configured write batch before +SQL persistence, reusing the write-path extraction, chunking, and model logic, and persists through +`UpdateResourceSearchParamsWithVectors` (v118). Version-conflicted resources are excluded from vector +deletion and insertion and remain eligible for a later cycle. `VectorSearchParameterResolver`'s indexing +view admits definitions that are enabled or explicitly `Supported`, so the same activation job can +backfill a posted definition before it becomes searchable. + +### 6.4 Linked-source refresh + +Because a `DocumentReference` and its `Binary` are separate resources that can change independently, +schema v119 adds durable refresh. Writes and hard deletes enqueue refresh work atomically at the owning +SQL procedure's commit boundary. `SqlVectorSearchSourceDependencyStore` resolves the owners that depend on +a changed source through the reverse-dependency index, and `VectorSearchSourceRefreshJob` reloads the +current owners, re-extracts and re-embeds their text, and persists the derived vectors without changing +the owner's FHIR version. Refresh job definitions retain the source version so `EnqueueJobs` +deduplication does not suppress later source updates; rapid source churn can produce redundant refresh +work and is an accepted operational limitation. + +--- + +## 7. Security and authorization + +- Ordinary FHIR resource authorization and the Patient compartment run first, so a caller only ranks and + receives resources it is permitted to see. Vectors live in the same database and inherit that model. +- Evidence authorization is fail-closed (`SemanticSearchEvidenceFilter`): a scored result is removed + entirely if any of its evidence sources (or, for chained queries, the witness) is denied, missing, + malformed, unsupported, or returns no authorized match. Survivors are reranked and `TotalCount` is + cleared whenever filtering changes membership, so no passage, score, or count leaks. +- Count-only exact totals are restricted for `LocalBinaryReference` vector parameters, because count-only + rows carry no source provenance to authorize; direct-text vector counts are unchanged. +- The embedding endpoint is reached with a managed identity (`DefaultAzureCredential`, no stored key). The + caller identity requires the Cognitive Services OpenAI User role. Note text sent for embedding stays + within the resource, region, and agreement that governs the endpoint. + +--- + +## 8. Component and dependency-injection map + +Registration happens in `Startup.AddSemanticSearch`. It is skipped, and no semantic services are +registered, unless the runtime data store is SQL Server and `VectorSearch.Enabled` is true. When active it +binds: + +| Abstraction | Implementation | Lifetime | +|---|---|---| +| `IVectorSearchParameterResolver` | `VectorSearchParameterResolver` | Singleton | +| `IEmbeddingClient` | `AzureFoundryEmbeddingClient` | Scoped | +| `IVectorStore` | `SqlVectorStore` | Scoped | +| `IEmbeddingModelRegistry` | `SqlEmbeddingModelRegistry` | Singleton | +| `IVectorSearchIndexer` | `VectorSearchIndexer` | Scoped | +| `IVectorSearchQueryProcessor` | `VectorSearchQueryProcessor` | Scoped | +| `ISemanticSearchEvidenceFilter` | `SemanticSearchEvidenceFilter` | Transient (SearchModule) | + +`DeterministicEmbeddingClient` is a deterministic stand-in used by tests and offline scenarios so ranking +behavior can be asserted without network access. + +Core abstractions (`Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch`): `IEmbeddingClient`, +`IEmbeddingModelRegistry`, `IVectorStore`, `IVectorSearchIndexer`, `IVectorSearchQueryProcessor`, +`IVectorSearchParameterResolver`, `IVectorSearchSourceDependencyStore`, `IVectorResourceReader`, +`IVectorTextSourceResolver`, `IBinaryContentExtractor`, `ITextChunker`, `ISemanticSearchEvidenceFilter`, +plus the value types `VectorSearchChunk`, `VectorSearchHit`, `VectorSearchResult`, +`VectorSearchIndexEntry`, `VectorTextSource`, `BinaryContentSegment`, `PreparedVectorSearchQuery`, +`PreparedVectorSearchChainLink`, and `SemanticSearchEvidence`. + +--- + +## 9. Testing + +- Core unit tests: configuration validation, evidence model and evidence filter, text chunker, embedding + client, vector indexer, parameter resolver, query processor, and text-source resolver + (`Microsoft.Health.Fhir.Core.UnitTests`, `Microsoft.Health.Fhir.Azure.UnitTests`). +- Shared/R4 tests: expression parsing (`VectorSearchExpressionParserTests`), parameter definition parsing, + `BundleFactory` score and evidence serialization, and the `SemanticSearchHandler` and + `SemanticSearchController` behavior. +- SQL Server tests: `SqlVectorStore`, `RemoveVectorSearchRewriter`, the source-refresh job and schema + gating, and the parameter validator. +- End-to-end: `SemanticSearchTests` in `Microsoft.Health.Fhir.R4.Tests.E2E` exercise the Patient operation + and compartment membership over Observation, DiagnosticReport, DocumentReference, and Coverage. E2E + execution requires a SQL Server 2025 instance with native `vector` support. + +Test runner note: this repo uses Microsoft Testing Platform. Pass filters after `--`, for example +`dotnet test -- --filter "FullyQualifiedName~SemanticSearch"`. Some environments discover zero +tests via `dotnet test`; running the built test dll directly is a reliable workaround. + +--- + +## 10. Build-time dependency and known limitation + +The `dbo.VectorSearchParam` table declares a `vector(1536)` column. Generating the SQL schema model +classes requires the shared `Microsoft.Health.Extensions.BuildTimeCodeGenerator` to recognize the `vector` +column type, which the released generator does not. The fix lives in an unreleased shared-components +change: [microsoft/healthcare-shared-components#1449](https://github.com/microsoft/healthcare-shared-components/pull/1449), +"Support VECTOR columns in generated SQL schema models," developed on branch +`users/t-annag/fix-vector-schema-model` (2 commits over tag `v11.0.128`). + +Until that ships, the branch builds only against a locally built preview package +(`1.0.0-fix-vector-schema-model-20260720-144944-preview`) served from a local NuGet feed, with +`HealthcareSharedPackageVersion` pinned to it for local and demo builds. That pin, and the machine-local +NuGet feed entry in `nuget.config`, must be reverted before any upstream pull request. To make the feature +build against released packages, decouple it from the generator (exclude the vector tables from generated +models and hand-author the row and table types, or model the column with a generator-supported type and +`CAST` to `vector(1536)` in SQL). See ADR-2608 for details. + +### 10.1 Current branch state and how to un-pin + +This branch is intentionally left pinned to the local preview package so it builds and demos as-is; +un-pinning is deliberately deferred to whoever finishes the feature. Two machine-local settings carry the +pin: + +- `Directory.Packages.props`: `HealthcareSharedPackageVersion` is set to + `1.0.0-fix-vector-schema-model-20260720-144944-preview` (for reference, `main` uses `11.0.135`). +- `nuget.config`: a `Local NuGet Feed` source points at a local `.nuget-local` directory that holds that + preview `Microsoft.Health.*` package set. + +To build this branch locally today, keep both settings and populate the local feed with the preview +package set. + +To un-pin once the generator fix ships in an official package (PR #1449 or an equivalent): + +1. In `Directory.Packages.props`, set `HealthcareSharedPackageVersion` to the official released version + that contains the fix (the same property `main` uses, for example a release after `11.0.135`). +2. In `nuget.config`, remove the `Local NuGet Feed` source and its `packageSourceMapping` entry. +3. Build against the released packages; no local feed is required. + +--- + +## 11. File inventory + +Semantic-search files introduced or changed by the branch, by area. Test files are omitted here for +brevity; see section 9. + +Core (`src/Microsoft.Health.Fhir.Core`): + +- `Configs/VectorSearchConfiguration.cs`, `VectorSearchEmbeddingConfiguration.cs`, + `VectorSearchIndexingConfiguration.cs`, `VectorSearchIndexingMode.cs`, `VectorSearchPdfConfiguration.cs`, + `VectorSearchQueryConfiguration.cs`. +- `Models/VectorSearchParameterConfig.cs`, `VectorTextExtractionPolicy.cs`, `VectorTextSourceStrategy.cs`, + and `SearchParameterInfo.cs` (adds `VectorConfig`). +- `Features/Search/SemanticSearch/` (all interfaces and implementations listed in section 8, plus + `VectorSearchIndexer.cs`, `VectorSearchQueryProcessor.cs`, `VectorSearchParameterResolver.cs`, + `VectorTextSourceResolver.cs`, `TextChunker.cs`, `PlainTextBinaryContentExtractor.cs`, + `PdfBinaryContentExtractor.cs`, `SemanticSearchEvidence.cs`, `SemanticSearchEvidenceFilter.cs`, + `SemanticSearchEvidenceRanker.cs`, `DeterministicEmbeddingClient.cs`). +- `Features/Search/Expressions/VectorSearchExpression.cs`, + `Features/Search/Expressions/Parsers/SearchParameterExpressionParser.cs`, + `Features/Search/SearchParameterInfoExtensions.cs`, `Features/Search/SearchParameterNames.cs`, + `Features/Search/SearchResultEntry.cs` (adds score and evidence). +- `Features/Persistence/ResourceWrapper.cs` (vector indices), `BundleWrappers/SearchParameterWrapper.cs` + (parses `vector-search-config`). +- `Features/Operations/Reindex/ReindexProcessingJob.cs`, + `Features/Operations/Reindex/VectorSearchSourceRefreshJobDefinition.cs`. +- `Messages/SemanticSearch/SemanticSearchRequest.cs`, `SemanticSearchResponse.cs`. +- `Data/OperationDefinition/semantic-search.json`. + +Shared Core (`src/Microsoft.Health.Fhir.Shared.Core`): + +- `Features/Search/SemanticSearch/SemanticSearchHandler.cs`, `Features/Search/BundleFactory.cs`, + `Features/Search/Parameters/SearchParameterToTypeResolver.cs` (supports `toString()`). + +Shared API and Web (`src/Microsoft.Health.Fhir.Shared.Api`, `src/Microsoft.Health.Fhir.Shared.Web`): + +- `Controllers/SemanticSearchController.cs`, `Startup.cs` (`AddSemanticSearch`). + +Azure (`src/Microsoft.Health.Fhir.Azure`): + +- `SemanticSearch/AzureFoundryEmbeddingClient.cs`. + +SQL Server (`src/Microsoft.Health.Fhir.SqlServer`): + +- `Features/Search/SemanticSearch/SqlVectorStore.cs`, `SqlDocumentReferenceSemanticSearch.cs`, + `SqlEmbeddingModelRegistry.cs`, `SqlVectorResourceReader.cs`, `SqlVectorFormatter.cs`. +- `Features/Storage/SqlServerFhirDataStore.cs`, `SqlVectorSearchSourceDependencyStore.cs`, + `TvpRowGeneration/Merge/VectorSearchParamListRowGenerator.cs`, + `TvpRowGeneration/Merge/MergeSearchParameterRowGenerator.cs`. +- `Features/Operations/VectorSearchSourceRefreshJob.cs`, + `Features/Search/Expressions/Visitors/RemoveVectorSearchRewriter.cs`, + `Features/Search/SqlServerSearchParameterValidator.cs`. +- `Features/Schema/SchemaVersion.cs`, `SchemaVersionConstants.cs`, and the SQL under + `Features/Schema/Sql/Tables/VectorSearchParam.sql`, `Sql/Types/VectorSearchParamList.sql`, and the + `Sql/Sprocs/*VectorSearch*` and merge procedures listed in section 5.2. + +--- + +## 12. References + +- ADR-2608, FHIR-Native SQL Semantic Search: [docs/arch/adr-2608-sql-semantic-search.md](arch/adr-2608-sql-semantic-search.md). +- Design spec, "Semantic Search over FHIR Clinical Documents" (health-paas-docs PR 65041). +- Build-time generator dependency: [microsoft/healthcare-shared-components#1449](https://github.com/microsoft/healthcare-shared-components/pull/1449). +- SQL schema versioning: [docs/SchemaVersioning.md](SchemaVersioning.md). +- Search architecture: [docs/SearchArchitecture.md](SearchArchitecture.md). diff --git a/docs/arch/adr-2608-sql-semantic-search.md b/docs/arch/adr-2608-sql-semantic-search.md new file mode 100644 index 0000000000..b3dfbc104f --- /dev/null +++ b/docs/arch/adr-2608-sql-semantic-search.md @@ -0,0 +1,62 @@ +# ADR-2608: FHIR-Native SQL Semantic Search + +**Status**: Proposed (first milestone implemented; SQL-only, disabled by default) +**Date**: 2026-08-23 +**Revised**: 2026-08-31 +**Feature**: SQL semantic search + +## Context + +FHIR search provides deterministic filtering but does not rank narrative chunks by conceptual similarity. Semantic retrieval must preserve FHIR authorization, patient and resource boundaries, SearchParameter lifecycle behavior, transactional resource consistency, and inspectable evidence. It must also support direct resource text and text held in a referenced local Binary without hard-coding resource types or parameter identities. + +The SQL implementation targets SQL Server 2025 native vectors and a fixed 1536-dimensional embedding contract. The initial workload is bounded by structured and Patient-compartment filters, so exact cosine ranking is preferred over a preview approximate index. The FHIR server returns retrieval results and evidence; clinical answer generation remains outside the server. + +## Options Considered + +1. **External vector database and proprietary retrieval API** - rejected: duplicates FHIR identity, authorization, lifecycle, and transaction boundaries. +2. **SQL vectors with resource-specific extraction code** - rejected: every new resource type or field would require server code and deployment. +3. **SearchParameter-driven SQL vectors integrated with FHIR search** - chosen: live FHIR metadata defines eligibility and extraction while the existing search pipeline preserves deterministic constraints. + +## Decision + +Use active, supported, searchable FHIR SearchParameter resources of type `special`, with a Microsoft vector configuration extension, as the source of truth for vector indexing and query behavior. Keep the feature SQL-only and disabled by default. Generate embeddings synchronously during normal writes, replace vectors transactionally with the current resource, support vector-aware `$reindex`, and enqueue durable owner refresh work when a referenced source changes or is deleted. + +Apply ordinary FHIR filters, Patient compartments, resource authorization, and linked evidence authorization before returning results. Rank the bounded candidate set with exact cosine distance in SQL Server 2025. Return whole FHIR resources with `Bundle.entry.search.score` and evidence containing the winning chunk, SearchParameter canonical, source path, versioned source, and an optional chained witness. Expose the capability through two surfaces: a `semantic-text` SearchParameter of type `special` bearing the Microsoft `vector-search-config` extension on ordinary resource search, and a Patient-scoped `$semantic-search` operation (`POST [base]/Patient/{id}/$semantic-search`) whose eligible types derive from Patient-compartment and vector SearchParameter metadata. Support one-level forward or reverse semantic chains, and carry per-match evidence in the `semantic-search-evidence` extension. Drive per-parameter extraction, source strategy (direct text or a referenced local Binary), chunking, minimum score, and distance metric from the `vector-search-config` extension; chunking defaults and query limits come from `VectorSearchConfiguration`. + +## Consequences + +- New semantic targets can be introduced through SearchParameter metadata and lifecycle operations instead of resource-type code changes. +- Structured filtering, authorization, vector ranking, source provenance, and resource persistence share the existing FHIR and SQL ownership boundaries. +- Synchronous embedding increases write latency and makes embedding availability part of successful-write availability. +- Local Binary references require Bundle dependency ordering, fail-closed evidence authorization, durable linked-source refresh jobs, and reindex queue hosting. +- The current storage contract is fixed at `vector(1536)` and cosine distance; changing dimensions requires schema and backfill work. +- Exact kNN is appropriate only while deterministic filters keep candidate sets bounded; approximate indexing remains future work. +- Cosmos DB, asynchronous indexing, multi-hop semantic chains, OCR, HAS-supplied code-metadata filtering, intent detection, and clinical answer generation are outside this decision. + +## Known Limitation — Build-Time Code Generation Dependency (Blocker) + +The SQL vector tables declare a `vector(1536)` column. Generating the SQL schema model classes requires `Microsoft.Health.Extensions.BuildTimeCodeGenerator` to recognize the `vector` column type. The released generator does not, so this feature currently depends on an unreleased change to the shared `healthcare-shared-components` code generator: PR [microsoft/healthcare-shared-components#1449](https://github.com/microsoft/healthcare-shared-components/pull/1449), "Support VECTOR columns in generated SQL schema models", developed on branch `users/t-annag/fix-vector-schema-model` (2 commits over tag `v11.0.128`). + +That shared-components change is not expected to merge or ship as an official package. Consequently: + +- The feature cannot be merged into `microsoft/fhir-server` main while it hard-depends on the unreleased generator, because upstream CI can only restore officially released `Microsoft.Health.*` packages. +- The branch builds only against a locally built preview package set (for example `1.0.0-fix-vector-schema-model-20260720-144944-preview`), built from PR #1449's branch and served from a local NuGet feed. `HealthcareSharedPackageVersion` is pointed at that preview for local and demo builds and must be reverted to an official version before any upstream PR. +- The feature is otherwise complete and demonstrable end-to-end against the local preview packages. + +### Recommended path to unblock (future work) + +Decouple the feature from the code generator so it builds with the released packages. Two options: + +- Exclude the vector tables from generated-model production and hand-author the required row and table model types, or +- Represent the vector column with a generator-supported type in the model layer and cast to `vector(1536)` in SQL at query and stored-procedure time. + +Either approach removes the dependency on the unreleased shared-components change and makes the feature independently mergeable. + +## References + +- Design spec: "Semantic Search over FHIR Clinical Documents" (`health-paas-docs`, PR 65041) — the in-depth companion to this ADR. +- ADR 2605: Vector Search Parameter. +- Build-time generator dependency: [microsoft/healthcare-shared-components#1449](https://github.com/microsoft/healthcare-shared-components/pull/1449), "Support VECTOR columns in generated SQL schema models" (branch `users/t-annag/fix-vector-schema-model`). +- `docs/SchemaVersioning.md`: SQL schema version and migration rules (this feature adds schema versions 117-119). +- SQL Server 2025 vector data type and `VECTOR_DISTANCE` (Microsoft Learn). +- Azure OpenAI / Azure AI Foundry embeddings (Microsoft Learn). \ No newline at end of file diff --git a/nuget.config b/nuget.config index f40187090d..386f475e93 100644 --- a/nuget.config +++ b/nuget.config @@ -6,6 +6,7 @@ + @@ -17,5 +18,11 @@ + + + + + + diff --git a/samples/templates/aca/fhir-sql.bicep b/samples/templates/aca/fhir-sql.bicep index 08da16e369..6e2d4f8f1f 100644 --- a/samples/templates/aca/fhir-sql.bicep +++ b/samples/templates/aca/fhir-sql.bicep @@ -27,10 +27,16 @@ param imageTag string = 'latest' @description('Existing SQL server name.') param sqlServerName string +@description('Existing SQL database name.') +param sqlDatabaseName string = 'FHIR${fhirVersion}' + @description('Schema automatic updates mode.') @allowed(['auto', 'tool']) param sqlSchemaAutomaticUpdatesEnabled string = 'auto' +@description('Delete all FHIR data when the application starts. Enable only for a controlled one-time reset.') +param deleteAllDataOnStartup bool = false + @description('Authority URL for AAD authentication.') param securityAuthenticationAuthority string = '' @@ -73,7 +79,6 @@ param additionalEnvVars array = [] var normalizedSqlServerName = toLower(sqlServerName) var sqlManagedIdentityName = '${normalizedSqlServerName}-uami' -var sqlDatabaseName = 'FHIR${fhirVersion}' var sqlManagedIdentityResourceId = resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', sqlManagedIdentityName) @@ -89,7 +94,7 @@ var datastoreEnvVars = [ name: 'SqlServer__SchemaOptions__AutomaticUpdatesEnabled' value: sqlSchemaAutomaticUpdatesEnabled == 'auto' ? 'true' : 'false' } - { name: 'SqlServer__DeleteAllDataOnStartup', value: 'false' } + { name: 'SqlServer__DeleteAllDataOnStartup', value: deleteAllDataOnStartup ? 'true' : 'false' } { name: 'SqlServer__AllowDatabaseCreation', value: 'true' } ] diff --git a/src/Microsoft.Health.Fhir.Azure.UnitTests/SemanticSearch/AzureFoundryEmbeddingClientTests.cs b/src/Microsoft.Health.Fhir.Azure.UnitTests/SemanticSearch/AzureFoundryEmbeddingClientTests.cs new file mode 100644 index 0000000000..62bfe7e1e5 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Azure.UnitTests/SemanticSearch/AzureFoundryEmbeddingClientTests.cs @@ -0,0 +1,51 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Identity; +using Microsoft.Health.Fhir.Azure.SemanticSearch; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Azure.UnitTests.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public class AzureFoundryEmbeddingClientTests + { + // Opt-in integration test: it only runs when the endpoint is configured through environment variables, + // so CI (which has neither credentials nor network access to the endpoint) stays offline. + // To run locally: az login, then set FHIR_TEST_EMBEDDING_ENDPOINT and FHIR_TEST_EMBEDDING_DEPLOYMENT. + [Fact] + public async Task GivenAConfiguredEndpoint_WhenEmbeddingText_ThenAVectorOfTheConfiguredDimensionsIsReturned() + { + string endpoint = Environment.GetEnvironmentVariable("FHIR_TEST_EMBEDDING_ENDPOINT"); + string deployment = Environment.GetEnvironmentVariable("FHIR_TEST_EMBEDDING_DEPLOYMENT"); + + if (string.IsNullOrWhiteSpace(endpoint) || string.IsNullOrWhiteSpace(deployment)) + { + return; + } + + var configuration = new VectorSearchEmbeddingConfiguration + { + Endpoint = new Uri(endpoint), + DeploymentName = deployment, + Dimensions = 1536, + }; + + var client = new AzureFoundryEmbeddingClient(configuration, new DefaultAzureCredential()); + + var embeddings = await client.GenerateEmbeddingsAsync(new[] { "chest pain" }, CancellationToken.None); + + Assert.Single(embeddings); + Assert.Equal(configuration.Dimensions, embeddings[0].Length); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Azure/Microsoft.Health.Fhir.Azure.csproj b/src/Microsoft.Health.Fhir.Azure/Microsoft.Health.Fhir.Azure.csproj index d34a8bd0b9..32c18748af 100644 --- a/src/Microsoft.Health.Fhir.Azure/Microsoft.Health.Fhir.Azure.csproj +++ b/src/Microsoft.Health.Fhir.Azure/Microsoft.Health.Fhir.Azure.csproj @@ -1,6 +1,7 @@  + diff --git a/src/Microsoft.Health.Fhir.Azure/SemanticSearch/AzureFoundryEmbeddingClient.cs b/src/Microsoft.Health.Fhir.Azure/SemanticSearch/AzureFoundryEmbeddingClient.cs new file mode 100644 index 0000000000..6e55bf8fad --- /dev/null +++ b/src/Microsoft.Health.Fhir.Azure/SemanticSearch/AzureFoundryEmbeddingClient.cs @@ -0,0 +1,71 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.ClientModel; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.AI.OpenAI; +using Azure.Core; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using OpenAI.Embeddings; + +namespace Microsoft.Health.Fhir.Azure.SemanticSearch +{ + /// + /// An that calls an external Azure OpenAI / Foundry embedding deployment. + /// It authenticates with a (managed identity in production, developer sign-in + /// locally), so no API key is ever stored. + /// + public sealed class AzureFoundryEmbeddingClient : IEmbeddingClient + { + private readonly EmbeddingClient _embeddingClient; + + /// + /// Initializes a new instance of the class. + /// + /// The embedding endpoint configuration. + /// The credential used to authenticate to the endpoint. + public AzureFoundryEmbeddingClient(VectorSearchEmbeddingConfiguration configuration, TokenCredential tokenCredential) + { + EnsureArg.IsNotNull(configuration, nameof(configuration)); + EnsureArg.IsNotNull(configuration.Endpoint, nameof(configuration.Endpoint)); + EnsureArg.IsNotNullOrWhiteSpace(configuration.DeploymentName, nameof(configuration.DeploymentName)); + EnsureArg.IsGt(configuration.Dimensions, 0, nameof(configuration.Dimensions)); + EnsureArg.IsNotNull(tokenCredential, nameof(tokenCredential)); + + Dimensions = configuration.Dimensions; + + var azureClient = new AzureOpenAIClient(configuration.Endpoint, tokenCredential); + _embeddingClient = azureClient.GetEmbeddingClient(configuration.DeploymentName); + } + + /// + public int Dimensions { get; } + + /// + public async Task> GenerateEmbeddingsAsync(IReadOnlyList texts, CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(texts, nameof(texts)); + + if (texts.Count == 0) + { + return Array.Empty(); + } + + var options = new EmbeddingGenerationOptions { Dimensions = Dimensions }; + + ClientResult response = await _embeddingClient.GenerateEmbeddingsAsync(texts, options, cancellationToken); + + return response.Value + .Select(embedding => embedding.ToFloats().ToArray()) + .ToList(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Config/VectorSearchConfigurationTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Config/VectorSearchConfigurationTests.cs new file mode 100644 index 0000000000..362692dc63 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Config/VectorSearchConfigurationTests.cs @@ -0,0 +1,406 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Config +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public sealed class VectorSearchConfigurationTests + { + [Fact] + public void GivenDefaultConfiguration_WhenCreated_ThenSafeDefaultsAreUsed() + { + // Arrange and Act + var configuration = new VectorSearchConfiguration(); + + // Assert + Assert.False(configuration.Enabled); + Assert.Null(configuration.Embedding.Endpoint); + Assert.Null(configuration.Embedding.DeploymentName); + Assert.Equal("text-embedding-3-small", configuration.Embedding.ModelName); + Assert.Null(configuration.Embedding.ModelVersion); + Assert.Equal(VectorSearchConfiguration.SupportedDimensions, configuration.Embedding.Dimensions); + Assert.Equal(VectorSearchIndexingMode.Synchronous, configuration.Indexing.Mode); + Assert.Equal(800, configuration.Indexing.ChunkSizeTokens); + Assert.Equal(100, configuration.Indexing.ChunkOverlapTokens); + Assert.Equal(10 * 1024 * 1024, configuration.Indexing.Pdf.MaximumFileSizeBytes); + Assert.Equal(200, configuration.Indexing.Pdf.MaximumPageCount); + Assert.Equal(500_000, configuration.Indexing.Pdf.MaximumExtractedCharacters); + Assert.Equal(TimeSpan.FromSeconds(30), configuration.Indexing.Pdf.ExtractionTimeout); + Assert.Equal(10, configuration.Query.DefaultCount); + Assert.Equal(50, configuration.Query.MaxCount); + Assert.Equal(100, configuration.Query.CandidateCount); + Assert.Equal(3, configuration.Query.EvidenceCount); + Assert.Equal(VectorSearchConfiguration.SupportedDistanceMetric, configuration.Query.DistanceMetric); + } + + [Fact] + public void GivenDisabledConfiguration_WhenInvalidValuesArePresent_ThenValidationIsSkipped() + { + // Arrange + var configuration = new VectorSearchConfiguration + { + Enabled = false, + Embedding = null, + Indexing = null, + Query = null, + }; + + // Act + Exception exception = Record.Exception(configuration.Validate); + + // Assert + Assert.Null(exception); + } + + [Fact] + public void GivenCompleteEnabledConfiguration_WhenValidated_ThenValidationSucceeds() + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + + // Act + Exception exception = Record.Exception(configuration.Validate); + + // Assert + Assert.Null(exception); + } + + [Theory] + [InlineData(null)] + [InlineData("/relative")] + [InlineData("http://embedding.example.com")] + public void GivenEnabledConfigurationWithInvalidEndpoint_WhenValidated_ThenValidationFails(string endpoint) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Embedding.Endpoint = endpoint == null ? null : new Uri(endpoint, UriKind.RelativeOrAbsolute); + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void GivenEnabledConfigurationWithMissingDeployment_WhenValidated_ThenValidationFails(string deploymentName) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Embedding.DeploymentName = deploymentName; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void GivenEnabledConfigurationWithMissingModel_WhenValidated_ThenValidationFails(string modelName) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Embedding.ModelName = modelName; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void GivenEnabledConfigurationWithMissingModelVersion_WhenValidated_ThenValidationFails(string modelVersion) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Embedding.ModelVersion = modelVersion; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(0)] + [InlineData(3072)] + public void GivenEnabledConfigurationWithUnsupportedDimensions_WhenValidated_ThenValidationFails(int dimensions) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Embedding.Dimensions = dimensions; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Fact] + public void GivenEnabledConfigurationWithoutEmbeddingSettings_WhenValidated_ThenValidationFails() + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Embedding = null; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Fact] + public void GivenEnabledConfigurationWithoutIndexingSettings_WhenValidated_ThenValidationFails() + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Indexing = null; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Fact] + public void GivenEnabledConfigurationWithUnsupportedIndexingMode_WhenValidated_ThenValidationFails() + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Indexing.Mode = (VectorSearchIndexingMode)int.MaxValue; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void GivenEnabledConfigurationWithInvalidChunkSize_WhenValidated_ThenValidationFails(int chunkSizeTokens) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Indexing.ChunkSizeTokens = chunkSizeTokens; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(-1)] + [InlineData(800)] + [InlineData(801)] + public void GivenEnabledConfigurationWithInvalidChunkOverlap_WhenValidated_ThenValidationFails(int chunkOverlapTokens) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Indexing.ChunkOverlapTokens = chunkOverlapTokens; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Fact] + public void GivenEnabledConfigurationWithoutPdfSettings_WhenValidated_ThenValidationFails() + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Indexing.Pdf = null; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void GivenEnabledConfigurationWithInvalidPdfFileSize_WhenValidated_ThenValidationFails(int maximumFileSizeBytes) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Indexing.Pdf.MaximumFileSizeBytes = maximumFileSizeBytes; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void GivenEnabledConfigurationWithInvalidPdfPageCount_WhenValidated_ThenValidationFails(int maximumPageCount) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Indexing.Pdf.MaximumPageCount = maximumPageCount; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void GivenEnabledConfigurationWithInvalidPdfCharacterLimit_WhenValidated_ThenValidationFails(int maximumExtractedCharacters) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Indexing.Pdf.MaximumExtractedCharacters = maximumExtractedCharacters; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void GivenEnabledConfigurationWithInvalidPdfTimeout_WhenValidated_ThenValidationFails(int timeoutSeconds) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Indexing.Pdf.ExtractionTimeout = TimeSpan.FromSeconds(timeoutSeconds); + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Fact] + public void GivenEnabledConfigurationWithoutQuerySettings_WhenValidated_ThenValidationFails() + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Query = null; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void GivenEnabledConfigurationWithInvalidDefaultCount_WhenValidated_ThenValidationFails(int defaultCount) + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Query.DefaultCount = defaultCount; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Fact] + public void GivenEnabledConfigurationWithMaximumBelowDefault_WhenValidated_ThenValidationFails() + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Query.DefaultCount = 10; + configuration.Query.MaxCount = 9; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Fact] + public void GivenEnabledConfigurationWithCandidateCountBelowMaximum_WhenValidated_ThenValidationFails() + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Query.CandidateCount = configuration.Query.MaxCount - 1; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + [Fact] + public void GivenEnabledConfigurationWithUnsupportedDistanceMetric_WhenValidated_ThenValidationFails() + { + // Arrange + VectorSearchConfiguration configuration = CreateValidConfiguration(); + configuration.Query.DistanceMetric = "euclidean"; + + // Act + Action validate = configuration.Validate; + + // Assert + Assert.Throws(validate); + } + + private static VectorSearchConfiguration CreateValidConfiguration() + { + var configuration = new VectorSearchConfiguration + { + Enabled = true, + Embedding = new VectorSearchEmbeddingConfiguration + { + Endpoint = new Uri("https://embedding.example.com"), + DeploymentName = "embedding-deployment", + ModelName = "text-embedding-3-small", + ModelVersion = "1", + Dimensions = VectorSearchConfiguration.SupportedDimensions, + }, + Indexing = new VectorSearchIndexingConfiguration(), + }; + + return configuration; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SearchParameterInfoExtensionsTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SearchParameterInfoExtensionsTests.cs index 0d9e16db8d..6859ba3bc5 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SearchParameterInfoExtensionsTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SearchParameterInfoExtensionsTests.cs @@ -123,7 +123,29 @@ public void GivenSearchParamsWithStatusEnabledOrDeleted_WhenCalculateSearchParam Assert.Equal(hash1, hash2); } - private SearchParameterInfo GenerateSearchParameterInfo(Uri uri, string resourceType, SortParameterStatus sortParameterStatus = SortParameterStatus.Disabled, SearchParameterStatus searchParameterStatus = SearchParameterStatus.Enabled) + [Fact] + public void GivenDifferentVectorConfiguration_WhenCalculateSearchParameterHash_ThenHashIsDifferent() + { + // Arrange + var firstConfig = new VectorSearchParameterConfig { MaxInputTokens = 1000 }; + var secondConfig = new VectorSearchParameterConfig { MaxInputTokens = 2000 }; + SearchParameterInfo first = GenerateSearchParameterInfo(_paramUri1, "Observation", vectorConfig: firstConfig); + SearchParameterInfo second = GenerateSearchParameterInfo(_paramUri1, "Observation", vectorConfig: secondConfig); + + // Act + string firstHash = new[] { first }.CalculateSearchParameterHash(); + string secondHash = new[] { second }.CalculateSearchParameterHash(); + + // Assert + Assert.NotEqual(firstHash, secondHash); + } + + private SearchParameterInfo GenerateSearchParameterInfo( + Uri uri, + string resourceType, + SortParameterStatus sortParameterStatus = SortParameterStatus.Disabled, + SearchParameterStatus searchParameterStatus = SearchParameterStatus.Enabled, + VectorSearchParameterConfig vectorConfig = null) { return new SearchParameterInfo( name: uri.Segments.LastOrDefault(), @@ -133,7 +155,8 @@ private SearchParameterInfo GenerateSearchParameterInfo(Uri uri, string resource components: null, expression: "expression", targetResourceTypes: null, - baseResourceTypes: new List { resourceType }) + baseResourceTypes: new List { resourceType }, + vectorConfig: vectorConfig) { SearchParameterStatus = searchParameterStatus, SortStatus = sortParameterStatus, diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/DeterministicEmbeddingClientTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/DeterministicEmbeddingClientTests.cs new file mode 100644 index 0000000000..2eafb8fb09 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/DeterministicEmbeddingClientTests.cs @@ -0,0 +1,80 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public class DeterministicEmbeddingClientTests + { + private readonly DeterministicEmbeddingClient _client = new DeterministicEmbeddingClient(dimensions: 1536); + + [Fact] + public async Task GivenTheSameText_WhenEmbeddedTwice_ThenTheVectorsAreIdentical() + { + var first = await _client.GenerateEmbeddingsAsync(new[] { "chest pain" }, CancellationToken.None); + var second = await _client.GenerateEmbeddingsAsync(new[] { "chest pain" }, CancellationToken.None); + + Assert.Equal(first[0], second[0]); + } + + [Fact] + public async Task GivenText_WhenEmbedded_ThenTheVectorHasTheConfiguredDimensions() + { + var embeddings = await _client.GenerateEmbeddingsAsync(new[] { "chest pain" }, CancellationToken.None); + + Assert.Equal(1536, embeddings[0].Length); + } + + [Fact] + public async Task GivenText_WhenEmbedded_ThenTheVectorIsL2Normalized() + { + var embeddings = await _client.GenerateEmbeddingsAsync(new[] { "chest pain" }, CancellationToken.None); + + double magnitude = Math.Sqrt(embeddings[0].Sum(component => (double)component * component)); + + Assert.Equal(1.0, magnitude, precision: 4); + } + + [Fact] + public async Task GivenDifferentText_WhenEmbedded_ThenTheVectorsDiffer() + { + var embeddings = await _client.GenerateEmbeddingsAsync(new[] { "chest pain", "broken arm" }, CancellationToken.None); + + Assert.NotEqual(embeddings[0], embeddings[1]); + } + + [Fact] + public async Task GivenMultipleTexts_WhenEmbedded_ThenOneVectorPerTextIsReturnedInOrder() + { + var texts = new[] { "a", "b", "c" }; + + var embeddings = await _client.GenerateEmbeddingsAsync(texts, CancellationToken.None); + + Assert.Equal(texts.Length, embeddings.Count); + } + + [Fact] + public async Task GivenNullTexts_WhenEmbedded_ThenArgumentNullExceptionIsThrown() + { + await Assert.ThrowsAsync(() => _client.GenerateEmbeddingsAsync(null, CancellationToken.None)); + } + + [Fact] + public void GivenNonPositiveDimensions_WhenConstructed_ThenArgumentExceptionIsThrown() + { + Assert.ThrowsAny(() => new DeterministicEmbeddingClient(dimensions: 0)); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/ModelInfoProviderSerialCollection.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/ModelInfoProviderSerialCollection.cs new file mode 100644 index 0000000000..8c7c4d6929 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/ModelInfoProviderSerialCollection.cs @@ -0,0 +1,18 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + /// + /// Serializes tests that replace the global ModelInfoProvider so they do not run in parallel with + /// tests that read it and would otherwise observe a partially configured provider. + /// + [CollectionDefinition(nameof(ModelInfoProviderSerialCollection), DisableParallelization = true)] + public sealed class ModelInfoProviderSerialCollection + { + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/PdfBinaryContentExtractorTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/PdfBinaryContentExtractorTests.cs new file mode 100644 index 0000000000..802180b026 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/PdfBinaryContentExtractorTests.cs @@ -0,0 +1,155 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using UglyToad.PdfPig.Content; +using UglyToad.PdfPig.Core; +using UglyToad.PdfPig.Fonts.Standard14Fonts; +using UglyToad.PdfPig.Writer; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public sealed class PdfBinaryContentExtractorTests + { + [Fact] + public void GivenTextPdf_WhenExtracting_ThenPageSegmentsAreReturnedInOrder() + { + // Arrange + PdfBinaryContentExtractor extractor = CreateExtractor(); + byte[] content = CreatePdf("first clinical page", "second clinical page"); + + // Act + bool extracted = extractor.TryExtract(content, "application/pdf", 1000, out IReadOnlyList segments); + + // Assert + Assert.True(extracted); + Assert.Collection( + segments, + segment => + { + Assert.Contains("first clinical page", segment.Text, StringComparison.Ordinal); + Assert.Equal("page=1", segment.SourceLocator); + }, + segment => + { + Assert.Contains("second clinical page", segment.Text, StringComparison.Ordinal); + Assert.Equal("page=2", segment.SourceLocator); + }); + } + + [Fact] + public void GivenPdfWithoutText_WhenExtracting_ThenExtractionFails() + { + // Arrange + PdfBinaryContentExtractor extractor = CreateExtractor(); + byte[] content = CreatePdf(new string[] { null }); + + // Act + bool extracted = extractor.TryExtract(content, "application/pdf", 1000, out IReadOnlyList segments); + + // Assert + Assert.False(extracted); + Assert.Null(segments); + } + + [Fact] + public void GivenMalformedPdf_WhenExtracting_ThenExtractionFails() + { + // Arrange + PdfBinaryContentExtractor extractor = CreateExtractor(); + + // Act + bool extracted = extractor.TryExtract(Encoding.UTF8.GetBytes("not a PDF"), "application/pdf", 1000, out IReadOnlyList segments); + + // Assert + Assert.False(extracted); + Assert.Null(segments); + } + + [Fact] + public void GivenPdfExceedingPageLimit_WhenExtracting_ThenExtractionFails() + { + // Arrange + var configuration = new VectorSearchConfiguration(); + configuration.Indexing.Pdf.MaximumPageCount = 1; + PdfBinaryContentExtractor extractor = CreateExtractor(configuration); + byte[] content = CreatePdf("first page", "second page"); + + // Act + bool extracted = extractor.TryExtract(content, "application/pdf", 1000, out IReadOnlyList segments); + + // Assert + Assert.False(extracted); + Assert.Null(segments); + } + + [Fact] + public void GivenPdfExceedingCharacterLimit_WhenExtracting_ThenExtractionFails() + { + // Arrange + var configuration = new VectorSearchConfiguration(); + configuration.Indexing.Pdf.MaximumExtractedCharacters = 5; + PdfBinaryContentExtractor extractor = CreateExtractor(configuration); + byte[] content = CreatePdf("clinical text"); + + // Act + bool extracted = extractor.TryExtract(content, "application/pdf", 1000, out IReadOnlyList segments); + + // Assert + Assert.False(extracted); + Assert.Null(segments); + } + + [Fact] + public void GivenPdfExceedingFileLimit_WhenExtracting_ThenExtractionFails() + { + // Arrange + byte[] content = CreatePdf("clinical text"); + var configuration = new VectorSearchConfiguration(); + configuration.Indexing.Pdf.MaximumFileSizeBytes = content.Length - 1; + PdfBinaryContentExtractor extractor = CreateExtractor(configuration); + + // Act + bool extracted = extractor.TryExtract(content, "application/pdf", 1000, out IReadOnlyList segments); + + // Assert + Assert.False(extracted); + Assert.Null(segments); + Assert.Equal(content.Length - 1, extractor.GetMaximumContentLength(maximumTextLength: 1)); + } + + private static PdfBinaryContentExtractor CreateExtractor(VectorSearchConfiguration configuration = null) + { + return new PdfBinaryContentExtractor(Options.Create(configuration ?? new VectorSearchConfiguration())); + } + + private static byte[] CreatePdf(params string[] pageTexts) + { + var builder = new PdfDocumentBuilder(); + PdfDocumentBuilder.AddedFont font = builder.AddStandard14Font(Standard14Font.Helvetica); + + foreach (string pageText in pageTexts) + { + PdfPageBuilder page = builder.AddPage(PageSize.A4); + if (pageText != null) + { + page.AddText(pageText, 12, new PdfPoint(25, 700), font); + } + } + + return builder.Build(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/SemanticSearchEvidenceFilterTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/SemanticSearchEvidenceFilterTests.cs new file mode 100644 index 0000000000..88e16233a4 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/SemanticSearchEvidenceFilterTests.cs @@ -0,0 +1,274 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Features; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public class SemanticSearchEvidenceFilterTests + { + private readonly ISearchService _searchService = Substitute.For(); + private readonly IDataResourceFilter _dataResourceFilter = Substitute.For(); + private readonly SemanticSearchEvidenceFilter _filter; + + public SemanticSearchEvidenceFilterTests() + { + _dataResourceFilter.Filter(Arg.Any()).Returns(callInfo => callInfo.Arg()); + _filter = new SemanticSearchEvidenceFilter(_searchService, _dataResourceFilter); + } + + [Fact] + public async Task GivenOwnerSourcedEvidence_WhenFiltered_ThenResultIsPreservedWithoutAdditionalSearch() + { + ResourceWrapper observation = CreateResourceWrapper("Observation", "observation", 1); + SearchResult searchResult = CreateSearchResult(CreateSemanticEntry(observation, "Observation/observation/_history/1", 0.9m)); + + SearchResult filtered = await _filter.FilterAsync(searchResult, CancellationToken.None); + + Assert.Same(searchResult, filtered); + await _searchService.DidNotReceiveWithAnyArgs().SearchAsync(default, default, default); + } + + [Fact] + public async Task GivenAuthorizedExternalSource_WhenFiltered_ThenResultAndEvidenceArePreserved() + { + ResourceWrapper owner = CreateResourceWrapper("DocumentReference", "document", 1); + ResourceWrapper source = CreateResourceWrapper("Binary", "source", 2); + SearchResult sourceSearchResult = CreateSearchResult(new SearchResultEntry(source)); + _searchService.SearchAsync( + "Binary", + Arg.Any>>(), + CancellationToken.None) + .Returns(sourceSearchResult); + + SearchResult filtered = await _filter.FilterAsync( + CreateSearchResult(CreateSemanticEntry(owner, "Binary/source/_history/1", 0.9m)), + CancellationToken.None); + + SearchResultEntry result = Assert.Single(filtered.Results); + Assert.Equal(0.9m, result.Score); + Assert.Equal(1, Assert.Single(result.EvidenceItems).Rank); + await _searchService.Received(1).SearchAsync( + "Binary", + Arg.Is>>(parameters => parameters.Single().Item1 == "_id" && parameters.Single().Item2 == "source"), + CancellationToken.None); + } + + [Fact] + public async Task GivenDeniedExternalSource_WhenFiltered_ThenResultScoreEvidenceAndTotalAreRemoved() + { + ResourceWrapper owner = CreateResourceWrapper("DocumentReference", "document", 1); + _searchService.SearchAsync( + "Binary", + Arg.Any>>(), + CancellationToken.None) + .Returns(SearchResult.Empty()); + + SearchResult filtered = await _filter.FilterAsync( + CreateSearchResult(CreateSemanticEntry(owner, "Binary/source/_history/1", 0.9m)), + CancellationToken.None); + + Assert.Empty(filtered.Results); + Assert.Null(filtered.TotalCount); + } + + [Fact] + public async Task GivenAuthorizedWitnessAndSource_WhenFiltered_ThenResultIsPreserved() + { + ResourceWrapper root = CreateResourceWrapper("Patient", "patient", 1); + ResourceWrapper witness = CreateResourceWrapper("DocumentReference", "document", 2); + ResourceWrapper source = CreateResourceWrapper("Binary", "source", 3); + _searchService.SearchAsync("DocumentReference", Arg.Any>>(), CancellationToken.None) + .Returns(CreateSearchResult(new SearchResultEntry(witness))); + _searchService.SearchAsync("Binary", Arg.Any>>(), CancellationToken.None) + .Returns(CreateSearchResult(new SearchResultEntry(source))); + + SearchResult filtered = await _filter.FilterAsync( + CreateSearchResult(CreateSemanticEntry( + root, + 0.9m, + CreateEvidence("Binary/source/_history/1", 0.9m, "DocumentReference/document/_history/1"))), + CancellationToken.None); + + Assert.Single(filtered.Results); + await _searchService.Received(1).SearchAsync("DocumentReference", Arg.Any>>(), CancellationToken.None); + await _searchService.Received(1).SearchAsync("Binary", Arg.Any>>(), CancellationToken.None); + } + + [Fact] + public async Task GivenDeniedWitnessAndAuthorizedSource_WhenFiltered_ThenWholeResultIsRemoved() + { + ResourceWrapper root = CreateResourceWrapper("Patient", "patient", 1); + ResourceWrapper source = CreateResourceWrapper("Binary", "source", 3); + _searchService.SearchAsync("DocumentReference", Arg.Any>>(), CancellationToken.None) + .Returns(SearchResult.Empty()); + _searchService.SearchAsync("Binary", Arg.Any>>(), CancellationToken.None) + .Returns(CreateSearchResult(new SearchResultEntry(source))); + + SearchResult filtered = await _filter.FilterAsync( + CreateSearchResult(CreateSemanticEntry( + root, + 0.9m, + CreateEvidence("Binary/source/_history/1", 0.9m, "DocumentReference/document/_history/1"))), + CancellationToken.None); + + Assert.Empty(filtered.Results); + } + + [Fact] + public async Task GivenMixedAuthorizedAndDeniedSources_WhenFiltered_ThenWholeResultIsRemoved() + { + ResourceWrapper owner = CreateResourceWrapper("DocumentReference", "document", 1); + ResourceWrapper allowedSource = CreateResourceWrapper("Binary", "allowed", 2); + _searchService.SearchAsync( + "Binary", + Arg.Any>>(), + CancellationToken.None) + .Returns(CreateSearchResult(new SearchResultEntry(allowedSource))); + SearchResultEntry result = CreateSemanticEntry( + owner, + 0.9m, + CreateEvidence("Binary/allowed/_history/1", 0.9m), + CreateEvidence("Binary/denied/_history/1", 0.8m)); + + SearchResult filtered = await _filter.FilterAsync(CreateSearchResult(result), CancellationToken.None); + + Assert.Empty(filtered.Results); + } + + [Theory] + [InlineData("https://example.org/fhir/Binary/source")] + [InlineData("Binary/source/_history")] + public async Task GivenInvalidSourceReference_WhenFiltered_ThenResultIsRemovedWithoutSourceSearch(string sourceReference) + { + ResourceWrapper owner = CreateResourceWrapper("DocumentReference", "document", 1); + + SearchResult filtered = await _filter.FilterAsync( + CreateSearchResult(CreateSemanticEntry(owner, sourceReference, 0.9m)), + CancellationToken.None); + + Assert.Empty(filtered.Results); + await _searchService.DidNotReceiveWithAnyArgs().SearchAsync(default, default, default); + } + + [Fact] + public async Task GivenInvalidWitnessReference_WhenFiltered_ThenResultIsRemovedWithoutSourceSearch() + { + ResourceWrapper root = CreateResourceWrapper("Patient", "patient", 1); + + SearchResult filtered = await _filter.FilterAsync( + CreateSearchResult(CreateSemanticEntry( + root, + 0.9m, + CreateEvidence("Binary/source/_history/1", 0.9m, "DocumentReference/document/_history"))), + CancellationToken.None); + + Assert.Empty(filtered.Results); + await _searchService.DidNotReceiveWithAnyArgs().SearchAsync(default, default, default); + } + + [Fact] + public async Task GivenUnsupportedSourceResourceType_WhenFiltered_ThenResultIsRemoved() + { + ResourceWrapper owner = CreateResourceWrapper("DocumentReference", "document", 1); + _searchService.SearchAsync( + "Unknown", + Arg.Any>>(), + CancellationToken.None) + .Returns>(_ => throw new ResourceNotSupportedException("Unknown")); + + SearchResult filtered = await _filter.FilterAsync( + CreateSearchResult(CreateSemanticEntry(owner, "Unknown/source", 0.9m)), + CancellationToken.None); + + Assert.Empty(filtered.Results); + } + + [Fact] + public async Task GivenUnauthorizedSourceSearch_WhenFiltered_ThenResultIsRemoved() + { + ResourceWrapper owner = CreateResourceWrapper("DocumentReference", "document", 1); + _searchService.SearchAsync( + "Binary", + Arg.Any>>(), + CancellationToken.None) + .Returns>(_ => throw new UnauthorizedFhirActionException()); + + SearchResult filtered = await _filter.FilterAsync( + CreateSearchResult(CreateSemanticEntry(owner, "Binary/source", 0.9m)), + CancellationToken.None); + + Assert.Empty(filtered.Results); + } + + private static SearchResultEntry CreateSemanticEntry(ResourceWrapper owner, string sourceReference, decimal score) + { + return CreateSemanticEntry(owner, score, CreateEvidence(sourceReference, score)); + } + + private static SearchResultEntry CreateSemanticEntry(ResourceWrapper owner, decimal score, params SemanticSearchEvidence[] evidence) + { + return new SearchResultEntry(owner, score: score, evidenceItems: evidence); + } + + private static SemanticSearchEvidence CreateEvidence(string sourceReference, decimal score, string witnessReference = null) + { + return new SemanticSearchEvidence( + "Matched passage", + chunkOrdinal: 0, + score, + new Uri("https://example.org/fhir/SearchParameter/semantic-text"), + sourceReference, + "Binary.data", + witnessReference: witnessReference); + } + + private static SearchResult CreateSearchResult(params SearchResultEntry[] results) + { + return new SearchResult( + results, + continuationToken: null, + sortOrder: null, + unsupportedSearchParameters: Array.Empty>()) + { + TotalCount = results.Length, + }; + } + + private static ResourceWrapper CreateResourceWrapper(string resourceType, string resourceId, long resourceSurrogateId) + { + return new ResourceWrapper( + resourceId, + versionId: "1", + resourceType, + new RawResource(new Lazy(() => $"{{\"resourceType\":\"{resourceType}\",\"id\":\"{resourceId}\"}}"), FhirResourceFormat.Json, isMetaSet: true), + new ResourceRequest(HttpMethod.Post, "http://test/resource"), + DateTimeOffset.UtcNow, + deleted: false, + searchIndices: null, + compartmentIndices: null, + lastModifiedClaims: null, + resourceSurrogateId: resourceSurrogateId); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/SemanticSearchEvidenceTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/SemanticSearchEvidenceTests.cs new file mode 100644 index 0000000000..d273d82ac7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/SemanticSearchEvidenceTests.cs @@ -0,0 +1,140 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public sealed class SemanticSearchEvidenceTests + { + [Fact] + public void GivenCompleteEvidence_WhenCreated_ThenPassageAndProvenanceArePreserved() + { + // Arrange + var canonical = new Uri("https://example.org/fhir/SearchParameter/observation-semantic-text"); + + // Act + var evidence = new SemanticSearchEvidence( + "The patient became short of breath while climbing stairs.", + chunkOrdinal: 2, + canonical, + "Observation/123/_history/4", + "Observation.note.text"); + + // Assert + Assert.Equal("The patient became short of breath while climbing stairs.", evidence.Text); + Assert.Equal(2, evidence.ChunkOrdinal); + Assert.Same(canonical, evidence.SearchParameterCanonical); + Assert.Equal("Observation/123/_history/4", evidence.SourceReference); + Assert.Equal("Observation.note.text", evidence.SourcePath); + Assert.Equal("http://microsoft.com/fhir/StructureDefinition/semantic-search-evidence", SemanticSearchEvidence.ExtensionUrl); + } + + [Fact] + public void GivenEvidenceFromReturnedResources_WhenAssigningRanks_ThenRanksAreDenseAcrossResources() + { + // Arrange + var canonical = new Uri("https://example.org/fhir/SearchParameter/semantic-text"); + IReadOnlyList> evidenceByResource = new[] + { + new[] + { + CreateEvidence("Resource one best", chunkOrdinal: 0, score: 0.90m, canonical), + CreateEvidence("Resource one second", chunkOrdinal: 1, score: 0.70m, canonical), + }, + new[] + { + CreateEvidence("Resource two best", chunkOrdinal: 0, score: 0.80m, canonical), + CreateEvidence("Resource two second", chunkOrdinal: 1, score: 0.70m, canonical), + }, + }; + + // Act + IReadOnlyList> ranked = SemanticSearchEvidenceRanker.AssignRanks(evidenceByResource); + + // Assert + Assert.Equal(new int?[] { 1, 3 }, ranked[0].Select(evidence => evidence.Rank)); + Assert.Equal(new int?[] { 2, 4 }, ranked[1].Select(evidence => evidence.Rank)); + Assert.All(evidenceByResource.SelectMany(evidence => evidence), evidence => Assert.Null(evidence.Rank)); + } + + [Fact] + public void GivenWitnessEvidence_WhenAssigningRank_ThenWitnessIsPreserved() + { + var evidence = new SemanticSearchEvidence( + "Matched Binary passage", + chunkOrdinal: 0, + score: 0.9m, + new Uri("https://example.org/fhir/SearchParameter/document-reference-semantic"), + "Binary/source/_history/2", + "Binary.data", + witnessReference: "DocumentReference/document/_history/3"); + + SemanticSearchEvidence ranked = Assert.Single(Assert.Single( + SemanticSearchEvidenceRanker.AssignRanks(new[] { new[] { evidence } }))); + + Assert.Equal("DocumentReference/document/_history/3", ranked.WitnessReference); + Assert.Equal(1, ranked.Rank); + } + + [Fact] + public void GivenRelativeSearchParameterCanonical_WhenCreatingEvidence_ThenArgumentExceptionIsThrown() + { + // Arrange + var relativeCanonical = new Uri("SearchParameter/observation-semantic-text", UriKind.Relative); + + // Act + Action create = () => new SemanticSearchEvidence( + "Matched passage", + chunkOrdinal: 0, + relativeCanonical, + "Observation/123/_history/4", + "Observation.note.text"); + + // Assert + Assert.Throws(create); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void GivenMissingPassageText_WhenCreatingEvidence_ThenArgumentExceptionIsThrown(string text) + { + // Arrange + var canonical = new Uri("https://example.org/fhir/SearchParameter/observation-semantic-text"); + + // Act + Action create = () => new SemanticSearchEvidence( + text, + chunkOrdinal: 0, + canonical, + "Observation/123/_history/4", + "Observation.note.text"); + + // Assert + Assert.ThrowsAny(create); + } + + private static SemanticSearchEvidence CreateEvidence(string text, int chunkOrdinal, decimal score, Uri canonical) + { + return new SemanticSearchEvidence( + text, + chunkOrdinal, + score, + canonical, + $"Observation/{chunkOrdinal}", + "Observation.note.text"); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/TextChunkerTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/TextChunkerTests.cs new file mode 100644 index 0000000000..6772ae02ce --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/TextChunkerTests.cs @@ -0,0 +1,83 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public class TextChunkerTests + { + private readonly TextChunker _chunker = new TextChunker(); + + [Fact] + public void GivenNullText_WhenChunked_ThenArgumentNullExceptionIsThrown() + { + Assert.Throws(() => _chunker.Chunk(null, chunkSize: 10, chunkOverlap: 2)); + } + + [Theory] + [InlineData(0, 0)] + [InlineData(-1, 0)] + public void GivenNonPositiveChunkSize_WhenChunked_ThenArgumentExceptionIsThrown(int chunkSize, int chunkOverlap) + { + Assert.ThrowsAny(() => _chunker.Chunk("some text", chunkSize, chunkOverlap)); + } + + [Theory] + [InlineData(4, 4)] + [InlineData(4, 5)] + [InlineData(4, -1)] + public void GivenInvalidOverlap_WhenChunked_ThenArgumentExceptionIsThrown(int chunkSize, int chunkOverlap) + { + Assert.ThrowsAny(() => _chunker.Chunk("some text", chunkSize, chunkOverlap)); + } + + [Fact] + public void GivenEmptyText_WhenChunked_ThenNoChunksAreReturned() + { + Assert.Empty(_chunker.Chunk(string.Empty, chunkSize: 10, chunkOverlap: 2)); + } + + [Fact] + public void GivenTextShorterThanChunkSize_WhenChunked_ThenASingleChunkEqualToTheTextIsReturned() + { + Assert.Equal(new[] { "short" }, _chunker.Chunk("short", chunkSize: 10, chunkOverlap: 2)); + } + + [Fact] + public void GivenTextEqualToChunkSize_WhenChunked_ThenASingleChunkIsReturned() + { + Assert.Equal(new[] { "abcd" }, _chunker.Chunk("abcd", chunkSize: 4, chunkOverlap: 1)); + } + + [Fact] + public void GivenOverlap_WhenChunked_ThenAdjacentChunksShareTheOverlap() + { + Assert.Equal(new[] { "abcd", "defg", "ghij" }, _chunker.Chunk("abcdefghij", chunkSize: 4, chunkOverlap: 1)); + } + + [Fact] + public void GivenNoOverlap_WhenChunked_ThenChunksArePartitionedWithoutSharing() + { + Assert.Equal(new[] { "abcde", "fghij" }, _chunker.Chunk("abcdefghij", chunkSize: 5, chunkOverlap: 0)); + } + + [Fact] + public void GivenAnyText_WhenChunked_ThenTheLastChunkEndsAtTheEndOfTheText() + { + const string text = "abcdefghijklmno"; + + var chunks = _chunker.Chunk(text, chunkSize: 6, chunkOverlap: 2); + + Assert.EndsWith(chunks[chunks.Count - 1], text); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/VectorSearchIndexerTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/VectorSearchIndexerTests.cs new file mode 100644 index 0000000000..96ed3a2330 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/VectorSearchIndexerTests.cs @@ -0,0 +1,287 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.ValueSets; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public sealed class VectorSearchIndexerTests + { + private static readonly Uri VectorCanonical = new Uri("https://example.org/fhir/SearchParameter/observation-note-vector"); + private static readonly Uri AlternateVectorCanonical = new Uri("https://example.org/fhir/SearchParameter/observation-text-vector"); + + [Fact] + public async Task GivenConcatenatePolicy_WhenIndexingExtractedValues_ThenOnePassageIsEmbeddedWithModelProvenance() + { + // Arrange + SearchParameterInfo searchParameter = CreateSearchParameter(VectorTextExtractionPolicy.Concatenate); + ResourceWrapper resource = CreateResource( + searchParameter, + new StringSearchValue("first value"), + new StringSearchValue("second value")); + var embeddedTexts = new List(); + VectorSearchIndexer indexer = CreateIndexer(searchParameter, embeddedTexts, embeddingModelId: 7); + + // Act + await indexer.IndexAsync(new[] { resource }, CancellationToken.None); + + // Assert + Assert.Equal(new[] { "first value\nsecond value" }, embeddedTexts); + VectorSearchIndexEntry indexEntry = Assert.Single(resource.VectorSearchIndices); + Assert.Same(searchParameter, indexEntry.SearchParameter); + Assert.Equal(7, indexEntry.EmbeddingModelId); + VectorSearchChunk chunk = Assert.Single(indexEntry.Chunks); + Assert.Equal(0, chunk.ChunkOrdinal); + Assert.Equal("first value\nsecond value", chunk.ChunkText); + Assert.Equal(32, chunk.SourceTextHash.Count); + } + + [Fact] + public async Task GivenPerValuePolicy_WhenValuesRequireChunking_ThenOrdinalsSpanAllExtractedValues() + { + // Arrange + SearchParameterInfo searchParameter = CreateSearchParameter(VectorTextExtractionPolicy.PerValueRow); + ResourceWrapper resource = CreateResource( + searchParameter, + new StringSearchValue("abcdef"), + new StringSearchValue("gh")); + var embeddedTexts = new List(); + VectorSearchIndexer indexer = CreateIndexer(searchParameter, embeddedTexts, chunkSize: 4); + + // Act + await indexer.IndexAsync(new[] { resource }, CancellationToken.None); + + // Assert + Assert.Equal(new[] { "abcd", "ef", "gh" }, embeddedTexts); + VectorSearchIndexEntry indexEntry = Assert.Single(resource.VectorSearchIndices); + Assert.Equal(new[] { 0, 1, 2 }, indexEntry.Chunks.Select(chunk => chunk.ChunkOrdinal)); + } + + [Fact] + public async Task GivenSearchParameterChunkSettings_WhenIndexing_ThenTheyOverrideGlobalDefaults() + { + // Arrange + SearchParameterInfo searchParameter = CreateSearchParameter( + VectorTextExtractionPolicy.Concatenate, + chunkSizeTokens: 4, + chunkOverlapTokens: 1); + ResourceWrapper resource = CreateResource(searchParameter, new StringSearchValue("abcdefghij")); + var embeddedTexts = new List(); + VectorSearchIndexer indexer = CreateIndexer(searchParameter, embeddedTexts, chunkSize: 10, chunkOverlap: 0); + + // Act + await indexer.IndexAsync(new[] { resource }, CancellationToken.None); + + // Assert + Assert.Equal(new[] { "abcd", "defg", "ghij" }, embeddedTexts); + } + + [Fact] + public async Task GivenNoSearchParameterChunkSettings_WhenIndexing_ThenGlobalDefaultsAreUsed() + { + // Arrange + SearchParameterInfo searchParameter = CreateSearchParameter(VectorTextExtractionPolicy.Concatenate); + ResourceWrapper resource = CreateResource(searchParameter, new StringSearchValue("abcdefghij")); + var embeddedTexts = new List(); + VectorSearchIndexer indexer = CreateIndexer(searchParameter, embeddedTexts, chunkSize: 5, chunkOverlap: 2); + + // Act + await indexer.IndexAsync(new[] { resource }, CancellationToken.None); + + // Assert + Assert.Equal(new[] { "abcde", "defgh", "ghij" }, embeddedTexts); + } + + [Fact] + public async Task GivenSearchParametersWithDifferentChunkSettings_WhenIndexingOneResource_ThenEachUsesItsOwnSettings() + { + // Arrange + SearchParameterInfo firstSearchParameter = CreateSearchParameter( + VectorTextExtractionPolicy.Concatenate, + chunkSizeTokens: 4, + chunkOverlapTokens: 0); + SearchParameterInfo secondSearchParameter = CreateSearchParameter( + VectorTextExtractionPolicy.Concatenate, + chunkSizeTokens: 5, + chunkOverlapTokens: 2, + canonical: AlternateVectorCanonical); + ResourceWrapper resource = CreateResource( + new SearchIndexEntry(firstSearchParameter, new StringSearchValue("abcdefgh")), + new SearchIndexEntry(secondSearchParameter, new StringSearchValue("ijklmnop"))); + var embeddedTexts = new List(); + VectorSearchIndexer indexer = CreateIndexer( + new[] { firstSearchParameter, secondSearchParameter }, + embeddedTexts, + chunkSize: 10, + chunkOverlap: 0); + + // Act + await indexer.IndexAsync(new[] { resource }, CancellationToken.None); + + // Assert + Assert.Equal(new[] { "abcd", "efgh", "ijklm", "lmnop" }, embeddedTexts); + Assert.Equal(2, resource.VectorSearchIndices.Count); + } + + [Fact] + public async Task GivenResourceWithoutEnabledSearchParameter_WhenIndexing_ThenEmbeddingServiceIsNotCalled() + { + // Arrange + ResourceWrapper resource = CreateResource(); + IVectorSearchParameterResolver resolver = Substitute.For(); + resolver.GetIndexingSearchParameters("Observation").Returns(Array.Empty()); + IEmbeddingClient embeddingClient = Substitute.For(); + var indexer = new VectorSearchIndexer( + resolver, + new TextChunker(), + embeddingClient, + Substitute.For(), + CreateTextSourceResolver(), + Options.Create(CreateConfiguration()), + NullLogger.Instance); + + // Act + await indexer.IndexAsync(new[] { resource }, CancellationToken.None); + + // Assert + Assert.Empty(resource.VectorSearchIndices); + Assert.True(resource.VectorSearchIndicesUpdated); + await embeddingClient.DidNotReceiveWithAnyArgs().GenerateEmbeddingsAsync(default, default); + } + + private static VectorSearchIndexer CreateIndexer( + SearchParameterInfo searchParameter, + List embeddedTexts, + short embeddingModelId = 1, + int chunkSize = 100, + int chunkOverlap = 0) + { + return CreateIndexer(new[] { searchParameter }, embeddedTexts, embeddingModelId, chunkSize, chunkOverlap); + } + + private static VectorSearchIndexer CreateIndexer( + IReadOnlyCollection searchParameters, + List embeddedTexts, + short embeddingModelId = 1, + int chunkSize = 100, + int chunkOverlap = 0) + { + IVectorSearchParameterResolver resolver = Substitute.For(); + resolver.GetIndexingSearchParameters("Observation").Returns(searchParameters); + + IEmbeddingClient embeddingClient = Substitute.For(); + embeddingClient.Dimensions.Returns(2); + embeddingClient.GenerateEmbeddingsAsync(Arg.Any>(), Arg.Any()) + .Returns(callInfo => + { + IReadOnlyList texts = callInfo.ArgAt>(0); + embeddedTexts.AddRange(texts); + IReadOnlyList embeddings = texts.Select(_ => new[] { 0.25f, 0.75f }).ToList(); + return Task.FromResult(embeddings); + }); + + IEmbeddingModelRegistry embeddingModelRegistry = Substitute.For(); + embeddingModelRegistry.GetEmbeddingModelIdAsync(Arg.Any()).Returns(embeddingModelId); + + VectorSearchConfiguration configuration = CreateConfiguration(); + configuration.Indexing.ChunkSizeTokens = chunkSize; + configuration.Indexing.ChunkOverlapTokens = chunkOverlap; + + return new VectorSearchIndexer( + resolver, + new TextChunker(), + embeddingClient, + embeddingModelRegistry, + CreateTextSourceResolver(), + Options.Create(configuration), + NullLogger.Instance); + } + + private static VectorTextSourceResolver CreateTextSourceResolver() + { + return new VectorTextSourceResolver( + Substitute.For(), + Substitute.For(), + new[] { new PlainTextBinaryContentExtractor() }); + } + + private static VectorSearchConfiguration CreateConfiguration() + { + return new VectorSearchConfiguration(); + } + + private static SearchParameterInfo CreateSearchParameter( + VectorTextExtractionPolicy extractionPolicy, + int? chunkSizeTokens = null, + int? chunkOverlapTokens = null, + Uri canonical = null) + { + return new SearchParameterInfo( + name: "ObservationNoteVector", + code: "note-vector", + searchParamType: SearchParamType.Special, + url: canonical ?? VectorCanonical, + expression: "Observation.note.text", + baseResourceTypes: new[] { "Observation" }, + vectorConfig: new VectorSearchParameterConfig + { + ExtractionPolicy = extractionPolicy, + ChunkSizeTokens = chunkSizeTokens, + ChunkOverlapTokens = chunkOverlapTokens, + }, + definitionStatus: "active"); + } + + private static ResourceWrapper CreateResource( + SearchParameterInfo searchParameter = null, + params StringSearchValue[] values) + { + IReadOnlyCollection searchIndices = searchParameter == null + ? Array.Empty() + : values.Select(value => new SearchIndexEntry(searchParameter, value)).ToList(); + + return CreateResource(searchIndices); + } + + private static ResourceWrapper CreateResource(params SearchIndexEntry[] searchIndices) + { + return CreateResource((IReadOnlyCollection)searchIndices); + } + + private static ResourceWrapper CreateResource(IReadOnlyCollection searchIndices) + { + return new ResourceWrapper( + resourceId: "example", + versionId: "1", + resourceTypeName: "Observation", + rawResource: new RawResource("{}", FhirResourceFormat.Json, isMetaSet: true), + request: new ResourceRequest("POST"), + lastModified: DateTimeOffset.UtcNow, + deleted: false, + searchIndices: searchIndices, + compartmentIndices: null, + lastModifiedClaims: Array.Empty>()); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/VectorSearchParameterResolverTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/VectorSearchParameterResolverTests.cs new file mode 100644 index 0000000000..63f380a0a4 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/VectorSearchParameterResolverTests.cs @@ -0,0 +1,189 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.ValueSets; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public sealed class VectorSearchParameterResolverTests + { + private static readonly Uri VectorCanonical = new Uri("https://example.org/fhir/SearchParameter/observation-note-vector"); + + [Fact] + public void GivenSearchableVectorSearchParameter_WhenResolvingResourceType_ThenOnlyVectorDefinitionIsReturned() + { + // Arrange + SearchParameterInfo vectorSearchParameter = CreateVectorSearchParameter(VectorCanonical); + var ordinarySearchParameter = new SearchParameterInfo( + "ObservationCode", + "code", + SearchParamType.Token, + new Uri("http://hl7.org/fhir/SearchParameter/Observation-code")); + ISearchParameterDefinitionManager definitionManager = Substitute.For(); + definitionManager.GetSearchParameters("Observation").Returns(new[] { ordinarySearchParameter, vectorSearchParameter }); + VectorSearchParameterResolver resolver = CreateResolver(definitionManager); + + // Act + IReadOnlyList results = resolver.GetSearchParameters("Observation"); + + // Assert + Assert.Collection(results, result => Assert.Same(vectorSearchParameter, result)); + } + + [Fact] + public void GivenUnregisteredCanonical_WhenResolved_ThenSearchParameterNotSupportedExceptionIsThrown() + { + // Arrange + var unregisteredCanonical = new Uri("https://example.org/fhir/SearchParameter/unregistered-vector"); + ISearchParameterDefinitionManager definitionManager = Substitute.For(); + VectorSearchParameterResolver resolver = CreateResolver(definitionManager); + + // Act + Action resolve = () => resolver.GetSearchParameter(unregisteredCanonical); + + // Assert + Assert.Throws(resolve); + } + + [Fact] + public void GivenVectorSearchParameterThatIsNotSearchable_WhenResolvingResourceType_ThenDefinitionIsSkipped() + { + // Arrange + SearchParameterInfo searchParameter = CreateVectorSearchParameter(VectorCanonical, isSearchable: false); + ISearchParameterDefinitionManager definitionManager = Substitute.For(); + definitionManager.GetSearchParameters("Observation").Returns(new[] { searchParameter }); + VectorSearchParameterResolver resolver = CreateResolver(definitionManager); + + // Act + IReadOnlyList results = resolver.GetSearchParameters("Observation"); + + // Assert + Assert.Empty(results); + } + + [Fact] + public void GivenSupportedVectorSearchParameterThatIsNotSearchable_WhenResolvingForIndexing_ThenDefinitionIsReturned() + { + // Arrange + SearchParameterInfo searchParameter = CreateVectorSearchParameter( + VectorCanonical, + isSearchable: false, + searchParameterStatus: SearchParameterStatus.Supported); + ISearchParameterDefinitionManager definitionManager = Substitute.For(); + definitionManager.GetSearchParameters("Observation").Returns(new[] { searchParameter }); + VectorSearchParameterResolver resolver = CreateResolver(definitionManager); + + // Act + IReadOnlyList results = resolver.GetIndexingSearchParameters("Observation"); + + // Assert + Assert.Collection(results, result => Assert.Same(searchParameter, result)); + } + + [Fact] + public void GivenNonSearchableVectorSearchParameterOutsideSupportedState_WhenResolvingForIndexing_ThenDefinitionIsSkipped() + { + // Arrange + SearchParameterInfo searchParameter = CreateVectorSearchParameter( + VectorCanonical, + isSearchable: false, + searchParameterStatus: SearchParameterStatus.Disabled); + ISearchParameterDefinitionManager definitionManager = Substitute.For(); + definitionManager.GetSearchParameters("Observation").Returns(new[] { searchParameter }); + VectorSearchParameterResolver resolver = CreateResolver(definitionManager); + + // Act + IReadOnlyList results = resolver.GetIndexingSearchParameters("Observation"); + + // Assert + Assert.Empty(results); + } + + [Fact] + public void GivenSearchParameterWithoutVectorExtension_WhenResolved_ThenResolutionFails() + { + // Arrange + SearchParameterInfo searchParameter = CreateVectorSearchParameter(VectorCanonical, includeVectorConfig: false); + ISearchParameterDefinitionManager definitionManager = Substitute.For(); + ConfigureRegisteredDefinition(definitionManager, searchParameter); + VectorSearchParameterResolver resolver = CreateResolver(definitionManager); + + // Act + Action resolve = () => resolver.GetSearchParameter(VectorCanonical); + + // Assert + Assert.Throws(resolve); + } + + [Fact] + public void GivenVectorSearchParameterThatIsNotActive_WhenResolved_ThenResolutionFails() + { + // Arrange + SearchParameterInfo searchParameter = CreateVectorSearchParameter(VectorCanonical, definitionStatus: "draft"); + ISearchParameterDefinitionManager definitionManager = Substitute.For(); + ConfigureRegisteredDefinition(definitionManager, searchParameter); + VectorSearchParameterResolver resolver = CreateResolver(definitionManager); + + // Act + Action resolve = () => resolver.GetSearchParameter(VectorCanonical); + + // Assert + Assert.Throws(resolve); + } + + private static VectorSearchParameterResolver CreateResolver(ISearchParameterDefinitionManager definitionManager) + { + return new VectorSearchParameterResolver(definitionManager); + } + + private static SearchParameterInfo CreateVectorSearchParameter( + Uri canonicalUri, + bool includeVectorConfig = true, + string definitionStatus = "active", + bool isSearchable = true, + SearchParameterStatus searchParameterStatus = SearchParameterStatus.Enabled) + { + var searchParameter = new SearchParameterInfo( + name: "ObservationNoteVector", + code: "note-vector", + searchParamType: SearchParamType.Special, + url: canonicalUri, + expression: "Observation.note.text", + baseResourceTypes: new[] { "Observation" }, + vectorConfig: includeVectorConfig ? new VectorSearchParameterConfig() : null, + definitionStatus: definitionStatus); + + searchParameter.IsSearchable = isSearchable; + searchParameter.IsSupported = true; + searchParameter.SearchParameterStatus = searchParameterStatus; + return searchParameter; + } + + private static void ConfigureRegisteredDefinition( + ISearchParameterDefinitionManager definitionManager, + SearchParameterInfo searchParameter) + { + definitionManager.TryGetSearchParameter(searchParameter.Url.OriginalString, true, out Arg.Any()) + .Returns(callInfo => + { + callInfo[2] = searchParameter; + return true; + }); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/VectorSearchQueryProcessorTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/VectorSearchQueryProcessorTests.cs new file mode 100644 index 0000000000..b7e76819fc --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/SemanticSearch/VectorSearchQueryProcessorTests.cs @@ -0,0 +1,260 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.ValueSets; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + [Collection(nameof(ModelInfoProviderSerialCollection))] + public sealed class VectorSearchQueryProcessorTests + { + private static readonly Uri VectorCanonical = new Uri("https://example.org/fhir/SearchParameter/semantic-text"); + private readonly IEmbeddingClient _embeddingClient = Substitute.For(); + private readonly IEmbeddingModelRegistry _embeddingModelRegistry = Substitute.For(); + + public VectorSearchQueryProcessorTests() + { + _embeddingClient.Dimensions.Returns(VectorSearchConfiguration.SupportedDimensions); + ModelInfoProvider.SetProvider( + MockModelInfoProviderBuilder.Create(FhirSpecification.R4) + .AddKnownTypes(KnownResourceTypes.Practitioner) + .Build()); + } + + [Fact] + public async Task GivenNoVectorExpression_WhenPreparing_ThenNoEmbeddingIsGenerated() + { + // Arrange + VectorSearchQueryProcessor processor = CreateProcessor(); + Expression expression = Expression.StringEquals(FieldName.String, null, "ordinary value", false); + + // Act + PreparedVectorSearchQuery result = await processor.PrepareAsync(expression, CancellationToken.None); + + // Assert + Assert.Null(result); + await _embeddingClient.DidNotReceiveWithAnyArgs().GenerateEmbeddingsAsync(default, default); + await _embeddingModelRegistry.DidNotReceiveWithAnyArgs().GetEmbeddingModelIdAsync(default); + } + + [Fact] + public async Task GivenOneVectorExpression_WhenPreparing_ThenEmbeddingAndModelProvenanceAreReturned() + { + // Arrange + const string queryText = "breathing difficulty overnight"; + const short embeddingModelId = 7; + using var cancellationTokenSource = new CancellationTokenSource(); + CancellationToken cancellationToken = cancellationTokenSource.Token; + float[] embedding = Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray(); + SearchParameterInfo searchParameter = CreateSearchParameter(); + _embeddingClient.GenerateEmbeddingsAsync(Arg.Any>(), cancellationToken) + .Returns(Task.FromResult>(new[] { embedding })); + _embeddingModelRegistry.GetEmbeddingModelIdAsync(cancellationToken).Returns(embeddingModelId); + VectorSearchQueryProcessor processor = CreateProcessor(); + + // Act + PreparedVectorSearchQuery result = await processor.PrepareAsync( + new VectorSearchExpression(searchParameter, queryText), + cancellationToken); + embedding[0] = 1.0f; + + // Assert + Assert.Same(searchParameter, result.SearchParameter); + Assert.Equal(embeddingModelId, result.EmbeddingModelId); + Assert.Equal(0.25f, result.Embedding[0]); + Assert.Equal(VectorSearchConfiguration.SupportedDimensions, result.Embedding.Count); + Assert.Equal(0.65m, result.MinimumScore); + Assert.Empty(result.ChainLinks); + await _embeddingClient.Received(1).GenerateEmbeddingsAsync( + Arg.Is>(texts => texts.SequenceEqual(new[] { queryText })), + cancellationToken); + await _embeddingModelRegistry.Received(1).GetEmbeddingModelIdAsync(cancellationToken); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GivenChainedVectorExpression_WhenPreparing_ThenRelationshipPathIsPreserved(bool reversed) + { + // Arrange + float[] embedding = Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray(); + _embeddingClient.GenerateEmbeddingsAsync(Arg.Any>(), Arg.Any()) + .Returns(Task.FromResult>(new[] { embedding })); + _embeddingModelRegistry.GetEmbeddingModelIdAsync(Arg.Any()).Returns((short)7); + var referenceSearchParameter = new SearchParameterInfo( + name: "subject", + code: "subject", + searchParamType: SearchParamType.Reference, + targetResourceTypes: new[] { KnownResourceTypes.Patient }); + var expression = new ChainedExpression( + new[] { KnownResourceTypes.Observation }, + referenceSearchParameter, + new[] { KnownResourceTypes.Patient }, + reversed, + CreateVectorExpression()); + + // Act + PreparedVectorSearchQuery result = await CreateProcessor().PrepareAsync(expression, CancellationToken.None); + + // Assert + PreparedVectorSearchChainLink chainLink = Assert.Single(result.ChainLinks); + Assert.Equal(new[] { KnownResourceTypes.Observation }, chainLink.ResourceTypes); + Assert.Same(referenceSearchParameter, chainLink.ReferenceSearchParameter); + Assert.Equal(new[] { KnownResourceTypes.Patient }, chainLink.TargetResourceTypes); + Assert.Equal(reversed, chainLink.Reversed); + } + + [Fact] + public async Task GivenMultipleVectorExpressions_WhenPreparing_ThenSearchIsRejectedBeforeEmbedding() + { + // Arrange + VectorSearchQueryProcessor processor = CreateProcessor(); + SearchParameterInfo searchParameter = CreateSearchParameter(); + Expression expression = Expression.And( + new VectorSearchExpression(searchParameter, "first query"), + new VectorSearchExpression(searchParameter, "second query")); + + // Act + Func prepare = () => processor.PrepareAsync(expression, CancellationToken.None); + + // Assert + await Assert.ThrowsAsync(prepare); + await _embeddingClient.DidNotReceiveWithAnyArgs().GenerateEmbeddingsAsync(default, default); + } + + [Fact] + public async Task GivenMultiHopVectorExpression_WhenPreparing_ThenSearchIsRejectedBeforeEmbedding() + { + var subjectSearchParameter = new SearchParameterInfo( + name: "subject", + code: "subject", + searchParamType: SearchParamType.Reference, + targetResourceTypes: new[] { KnownResourceTypes.Patient }); + var generalPractitionerSearchParameter = new SearchParameterInfo( + name: "general-practitioner", + code: "general-practitioner", + searchParamType: SearchParamType.Reference, + targetResourceTypes: new[] { KnownResourceTypes.Practitioner }); + var expression = new ChainedExpression( + new[] { KnownResourceTypes.Observation }, + subjectSearchParameter, + new[] { KnownResourceTypes.Patient }, + reversed: false, + new ChainedExpression( + new[] { KnownResourceTypes.Patient }, + generalPractitionerSearchParameter, + new[] { KnownResourceTypes.Practitioner }, + reversed: false, + CreateVectorExpression())); + + await Assert.ThrowsAsync( + () => CreateProcessor().PrepareAsync(expression, CancellationToken.None)); + await _embeddingClient.DidNotReceiveWithAnyArgs().GenerateEmbeddingsAsync(default, default); + } + + [Fact] + public async Task GivenLinkedSourceChainedVectorExpression_WhenPreparing_ThenWitnessPathIsPreserved() + { + float[] embedding = Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray(); + _embeddingClient.GenerateEmbeddingsAsync(Arg.Any>(), Arg.Any()) + .Returns(Task.FromResult>(new[] { embedding })); + _embeddingModelRegistry.GetEmbeddingModelIdAsync(Arg.Any()).Returns((short)7); + var referenceSearchParameter = new SearchParameterInfo( + name: "subject", + code: "subject", + searchParamType: SearchParamType.Reference, + targetResourceTypes: new[] { KnownResourceTypes.Patient }); + SearchParameterInfo vectorSearchParameter = CreateSearchParameter( + new VectorSearchParameterConfig { SourceStrategy = VectorTextSourceStrategy.LocalBinaryReference }); + var expression = new ChainedExpression( + new[] { KnownResourceTypes.Observation }, + referenceSearchParameter, + new[] { KnownResourceTypes.Patient }, + reversed: false, + new VectorSearchExpression(vectorSearchParameter, "breathing difficulty")); + + PreparedVectorSearchQuery result = await CreateProcessor().PrepareAsync(expression, CancellationToken.None); + + Assert.Same(vectorSearchParameter, result.SearchParameter); + PreparedVectorSearchChainLink chainLink = Assert.Single(result.ChainLinks); + Assert.Same(referenceSearchParameter, chainLink.ReferenceSearchParameter); + await _embeddingClient.Received(1).GenerateEmbeddingsAsync( + Arg.Is>(texts => texts.SequenceEqual(new[] { "breathing difficulty" })), + CancellationToken.None); + } + + [Fact] + public async Task GivenEmbeddingServiceReturnsWrongCount_WhenPreparing_ThenPreparationFails() + { + // Arrange + _embeddingClient.GenerateEmbeddingsAsync(Arg.Any>(), Arg.Any()) + .Returns(Task.FromResult>(Array.Empty())); + VectorSearchQueryProcessor processor = CreateProcessor(); + + // Act + Func prepare = () => processor.PrepareAsync(CreateVectorExpression(), CancellationToken.None); + + // Assert + await Assert.ThrowsAsync(prepare); + await _embeddingModelRegistry.DidNotReceiveWithAnyArgs().GetEmbeddingModelIdAsync(default); + } + + [Fact] + public async Task GivenEmbeddingServiceReturnsWrongDimensions_WhenPreparing_ThenPreparationFails() + { + // Arrange + var embedding = new float[VectorSearchConfiguration.SupportedDimensions - 1]; + _embeddingClient.GenerateEmbeddingsAsync(Arg.Any>(), Arg.Any()) + .Returns(Task.FromResult>(new[] { embedding })); + VectorSearchQueryProcessor processor = CreateProcessor(); + + // Act + Func prepare = () => processor.PrepareAsync(CreateVectorExpression(), CancellationToken.None); + + // Assert + await Assert.ThrowsAsync(prepare); + await _embeddingModelRegistry.DidNotReceiveWithAnyArgs().GetEmbeddingModelIdAsync(default); + } + + private VectorSearchQueryProcessor CreateProcessor() + { + return new VectorSearchQueryProcessor(_embeddingClient, _embeddingModelRegistry); + } + + private static VectorSearchExpression CreateVectorExpression() + { + return new VectorSearchExpression(CreateSearchParameter(), "breathing difficulty"); + } + + private static SearchParameterInfo CreateSearchParameter(VectorSearchParameterConfig vectorConfig = null) + { + return new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: VectorCanonical, + expression: "Resource.text.div", + baseResourceTypes: new[] { "Resource" }, + vectorConfig: vectorConfig ?? new VectorSearchParameterConfig { MinimumScore = 0.65m }, + definitionStatus: "active"); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs index 8c6c408aab..c67934f057 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs @@ -96,6 +96,11 @@ public VersioningConfiguration Versioning } } + /// + /// Gets the vector search configuration. + /// + public VectorSearchConfiguration VectorSearch { get; } = new VectorSearchConfiguration(); + /// /// Gets or sets a value indicating whether the server supports the $status operation for SearchParameters. /// diff --git a/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchConfiguration.cs new file mode 100644 index 0000000000..b34cafc829 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchConfiguration.cs @@ -0,0 +1,162 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Fhir.Core.Configs +{ + /// + /// Configures vector generation, persistence, and query limits for semantic search. + /// + public sealed class VectorSearchConfiguration + { + /// + /// The vector width supported by the current SQL schema. + /// + public const int SupportedDimensions = 1536; + + /// + /// The distance metric supported by the current semantic score calculation. + /// + public const string SupportedDistanceMetric = "cosine"; + + /// + /// Gets or sets a value indicating whether vector search is enabled. + /// + public bool Enabled { get; set; } + + /// + /// Gets or sets the embedding service configuration. + /// + public VectorSearchEmbeddingConfiguration Embedding { get; set; } = new VectorSearchEmbeddingConfiguration(); + + /// + /// Gets or sets the vector indexing configuration. + /// + public VectorSearchIndexingConfiguration Indexing { get; set; } = new VectorSearchIndexingConfiguration(); + + /// + /// Gets or sets the vector query configuration. + /// + public VectorSearchQueryConfiguration Query { get; set; } = new VectorSearchQueryConfiguration(); + + /// + /// Validates settings that are required when vector search is enabled. + /// + /// The enabled configuration is incomplete or incompatible. + public void Validate() + { + if (!Enabled) + { + return; + } + + if (Embedding == null) + { + throw new InvalidOperationException("Vector search embedding configuration is required when vector search is enabled."); + } + + if (Embedding.Endpoint == null || !Embedding.Endpoint.IsAbsoluteUri || !string.Equals(Embedding.Endpoint.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Vector search embedding endpoint must be an absolute HTTPS URI when vector search is enabled."); + } + + if (string.IsNullOrWhiteSpace(Embedding.DeploymentName)) + { + throw new InvalidOperationException("Vector search embedding deployment name is required when vector search is enabled."); + } + + if (string.IsNullOrWhiteSpace(Embedding.ModelName)) + { + throw new InvalidOperationException("Vector search embedding model name is required when vector search is enabled."); + } + + if (string.IsNullOrWhiteSpace(Embedding.ModelVersion)) + { + throw new InvalidOperationException("Vector search embedding model version is required when vector search is enabled."); + } + + if (Embedding.Dimensions != SupportedDimensions) + { + throw new InvalidOperationException($"Vector search embedding dimensions must be {SupportedDimensions} to match the current SQL vector schema."); + } + + if (Indexing == null) + { + throw new InvalidOperationException("Vector search indexing configuration is required when vector search is enabled."); + } + + if (Indexing.Mode != VectorSearchIndexingMode.Synchronous) + { + throw new InvalidOperationException($"Vector search indexing mode '{Indexing.Mode}' is not supported."); + } + + if (Indexing.ChunkSizeTokens <= 0) + { + throw new InvalidOperationException("Vector search chunk size must be greater than zero."); + } + + if (Indexing.ChunkOverlapTokens < 0 || Indexing.ChunkOverlapTokens >= Indexing.ChunkSizeTokens) + { + throw new InvalidOperationException("Vector search chunk overlap must be non-negative and smaller than the chunk size."); + } + + if (Indexing.Pdf == null) + { + throw new InvalidOperationException("Vector search PDF extraction configuration is required when vector search is enabled."); + } + + if (Indexing.Pdf.MaximumFileSizeBytes <= 0) + { + throw new InvalidOperationException("Vector search PDF maximum file size must be greater than zero."); + } + + if (Indexing.Pdf.MaximumPageCount <= 0) + { + throw new InvalidOperationException("Vector search PDF maximum page count must be greater than zero."); + } + + if (Indexing.Pdf.MaximumExtractedCharacters <= 0) + { + throw new InvalidOperationException("Vector search PDF maximum extracted characters must be greater than zero."); + } + + if (Indexing.Pdf.ExtractionTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Vector search PDF extraction timeout must be greater than zero."); + } + + if (Query == null) + { + throw new InvalidOperationException("Vector search query configuration is required when vector search is enabled."); + } + + if (Query.DefaultCount <= 0) + { + throw new InvalidOperationException("Vector search default result count must be greater than zero."); + } + + if (Query.MaxCount < Query.DefaultCount) + { + throw new InvalidOperationException("Vector search maximum result count must be greater than or equal to the default result count."); + } + + if (Query.CandidateCount < Query.MaxCount) + { + throw new InvalidOperationException("Vector search candidate count must be greater than or equal to the maximum result count."); + } + + if (Query.EvidenceCount <= 0) + { + throw new InvalidOperationException("Vector search evidence count must be greater than zero."); + } + + if (!string.Equals(Query.DistanceMetric, SupportedDistanceMetric, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"Vector search distance metric must be '{SupportedDistanceMetric}'."); + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchEmbeddingConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchEmbeddingConfiguration.cs new file mode 100644 index 0000000000..46c00ac3ea --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchEmbeddingConfiguration.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Fhir.Core.Configs +{ + /// + /// Configures the embedding service used to generate vectors. + /// + public sealed class VectorSearchEmbeddingConfiguration + { + /// + /// Gets or sets the embedding service endpoint. + /// + public Uri Endpoint { get; set; } + + /// + /// Gets or sets the embedding deployment name. + /// + public string DeploymentName { get; set; } + + /// + /// Gets or sets the embedding model name recorded as vector provenance. + /// + public string ModelName { get; set; } = "text-embedding-3-small"; + + /// + /// Gets or sets the embedding model version recorded as vector provenance. + /// + public string ModelVersion { get; set; } + + /// + /// Gets or sets the number of dimensions produced for each embedding. + /// + public int Dimensions { get; set; } = VectorSearchConfiguration.SupportedDimensions; + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchIndexingConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchIndexingConfiguration.cs new file mode 100644 index 0000000000..a44a4a1df2 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchIndexingConfiguration.cs @@ -0,0 +1,33 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Configs +{ + /// + /// Configures how vectors are generated when resources are indexed. + /// + public sealed class VectorSearchIndexingConfiguration + { + /// + /// Gets or sets the vector indexing mode. + /// + public VectorSearchIndexingMode Mode { get; set; } = VectorSearchIndexingMode.Synchronous; + + /// + /// Gets or sets the maximum number of tokens in an embedding chunk. + /// + public int ChunkSizeTokens { get; set; } = 800; + + /// + /// Gets or sets the number of tokens repeated between adjacent chunks. + /// + public int ChunkOverlapTokens { get; set; } = 100; + + /// + /// Gets or sets PDF text extraction settings. + /// + public VectorSearchPdfConfiguration Pdf { get; set; } = new VectorSearchPdfConfiguration(); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchIndexingMode.cs b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchIndexingMode.cs new file mode 100644 index 0000000000..58dcb56d3c --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchIndexingMode.cs @@ -0,0 +1,18 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Configs +{ + /// + /// Specifies when embeddings are generated for indexed resources. + /// + public enum VectorSearchIndexingMode + { + /// + /// Generates embeddings in the resource write path. + /// + Synchronous, + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchPdfConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchPdfConfiguration.cs new file mode 100644 index 0000000000..35e54f3fcb --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchPdfConfiguration.cs @@ -0,0 +1,35 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Fhir.Core.Configs +{ + /// + /// Configures resource limits for extracting text from PDF Binary content. + /// + public sealed class VectorSearchPdfConfiguration + { + /// + /// Gets or sets the maximum decoded PDF size in bytes. + /// + public int MaximumFileSizeBytes { get; set; } = 10 * 1024 * 1024; + + /// + /// Gets or sets the maximum number of pages allowed in a PDF. + /// + public int MaximumPageCount { get; set; } = 200; + + /// + /// Gets or sets the maximum total number of characters extracted from a PDF. + /// + public int MaximumExtractedCharacters { get; set; } = 500_000; + + /// + /// Gets or sets the maximum elapsed time allowed for PDF text extraction. + /// + public TimeSpan ExtractionTimeout { get; set; } = TimeSpan.FromSeconds(30); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchQueryConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchQueryConfiguration.cs new file mode 100644 index 0000000000..8fe18772c6 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Configs/VectorSearchQueryConfiguration.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Configs +{ + /// + /// Configures vector search result limits. + /// + public sealed class VectorSearchQueryConfiguration + { + /// + /// Gets or sets the default number of semantic search results. + /// + public int DefaultCount { get; set; } = 10; + + /// + /// Gets or sets the maximum number of semantic search results. + /// + public int MaxCount { get; set; } = 50; + + /// + /// Gets or sets the number of structured-search candidates considered for semantic ranking. + /// + public int CandidateCount { get; set; } = 100; + + /// + /// Gets or sets the maximum number of ranked evidence passages returned for each matched resource. + /// + public int EvidenceCount { get; set; } = 3; + + /// + /// Gets or sets the vector distance metric. + /// + public string DistanceMetric { get; set; } = VectorSearchConfiguration.SupportedDistanceMetric; + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Data/OperationDefinition/semantic-search.json b/src/Microsoft.Health.Fhir.Core/Data/OperationDefinition/semantic-search.json new file mode 100644 index 0000000000..65aaa2db28 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Data/OperationDefinition/semantic-search.json @@ -0,0 +1,56 @@ +{ + "resourceType": "OperationDefinition", + "id": "semantic-search", + "url": "[base]/OperationDefinition/semantic-search", + "version": "1.0.0", + "name": "SemanticSearch", + "title": "Patient Semantic Search", + "status": "active", + "kind": "operation", + "experimental": true, + "date": "2026-07-28", + "publisher": "Microsoft", + "description": "Searches configured resource types in a patient compartment by semantic similarity while preserving authorization and ordinary FHIR scope constraints.", + "affectsState": false, + "code": "semantic-search", + "resource": [ + "Patient" + ], + "system": false, + "type": false, + "instance": true, + "parameter": [ + { + "name": "query", + "use": "in", + "min": 1, + "max": "1", + "documentation": "Natural-language text used to rank semantically similar resources in the patient compartment.", + "type": "string" + }, + { + "name": "type", + "use": "in", + "min": 0, + "max": "*", + "documentation": "Optional resource type to include. Repeating this parameter forms a union of supported resource types.", + "type": "code" + }, + { + "name": "count", + "use": "in", + "min": 0, + "max": "1", + "documentation": "Maximum number of globally ranked results to return.", + "type": "integer" + }, + { + "name": "return", + "use": "out", + "min": 1, + "max": "1", + "documentation": "A searchset Bundle containing globally ranked resources with semantic evidence.", + "type": "Bundle" + } + ] +} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Core/Features/Definition/BundleWrappers/SearchParameterWrapper.cs b/src/Microsoft.Health.Fhir.Core/Features/Definition/BundleWrappers/SearchParameterWrapper.cs index 7686816232..ef5b742d2f 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Definition/BundleWrappers/SearchParameterWrapper.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Definition/BundleWrappers/SearchParameterWrapper.cs @@ -5,10 +5,13 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using EnsureThat; using Hl7.Fhir.ElementModel; using Hl7.FhirPath; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Models; @@ -24,7 +27,9 @@ internal class SearchParameterWrapper private readonly Lazy _code; private readonly Lazy> _target; private readonly Lazy _description; + private readonly Lazy _status; private Lazy _type; + private readonly Lazy _vectorConfig; public SearchParameterWrapper(ITypedElement searchParameter) { @@ -34,6 +39,7 @@ public SearchParameterWrapper(ITypedElement searchParameter) _name = new Lazy(() => searchParameter.Scalar("name")?.ToString()); _code = new Lazy(() => searchParameter.Scalar("code")?.ToString()); _description = new Lazy(() => searchParameter.Scalar("description")?.ToString()); + _status = new Lazy(() => searchParameter.Scalar("status")?.ToString()); _url = new Lazy(() => searchParameter.Scalar("url")?.ToString()); _expression = new Lazy(() => searchParameter.Scalar("expression")?.ToString()); _type = new Lazy(() => searchParameter.Scalar("type")?.ToString()); @@ -41,6 +47,7 @@ public SearchParameterWrapper(ITypedElement searchParameter) _base = new Lazy>(() => searchParameter.Select("base")?.AsStringValues().ToArray()); _component = new Lazy>(() => searchParameter.Select("component")?.ToArray()); _target = new Lazy>(() => searchParameter.Select("target")?.AsStringValues().ToArray()); + _vectorConfig = new Lazy(() => ParseVectorConfig(searchParameter.Select("extension"))); } public string Name => _name.Value; @@ -49,6 +56,8 @@ public SearchParameterWrapper(ITypedElement searchParameter) public string Description => _description.Value; + public string Status => _status.Value; + #pragma warning disable CA1056 // URI-like properties should not be strings public string Url => _url.Value; #pragma warning restore CA1056 // URI-like properties should not be strings @@ -62,5 +71,110 @@ public SearchParameterWrapper(ITypedElement searchParameter) public IReadOnlyList Target => _target.Value; public IReadOnlyList Component => _component.Value; + + public VectorSearchParameterConfig VectorConfig => _vectorConfig.Value; + + private static VectorSearchParameterConfig ParseVectorConfig(IEnumerable extensions) + { + ITypedElement[] vectorExtensions = extensions + .Where(extension => string.Equals( + extension.Scalar("url")?.ToString(), + VectorSearchParameterConfig.ExtensionUrl, + StringComparison.Ordinal)) + .ToArray(); + + if (vectorExtensions.Length == 0) + { + return null; + } + + if (vectorExtensions.Length > 1) + { + throw new InvalidDefinitionException($"SearchParameter contains multiple '{VectorSearchParameterConfig.ExtensionUrl}' extensions."); + } + + var configuration = new VectorSearchParameterConfig(); + foreach (ITypedElement nestedExtension in vectorExtensions[0].Select("extension")) + { + string url = nestedExtension.Scalar("url")?.ToString(); + object value = nestedExtension.Scalar("value") ?? + nestedExtension.Scalar("valueCode") ?? + nestedExtension.Scalar("valueInteger") ?? + nestedExtension.Scalar("valueDecimal"); + + if (string.Equals(url, VectorSearchParameterConfig.ExtractionPolicyExtensionUrl, StringComparison.Ordinal)) + { + if (!Enum.TryParse(value?.ToString(), ignoreCase: true, out VectorTextExtractionPolicy extractionPolicy)) + { + throw new InvalidDefinitionException($"Vector SearchParameter extraction policy '{value}' is not supported."); + } + + configuration.ExtractionPolicy = extractionPolicy; + } + else if (string.Equals(url, VectorSearchParameterConfig.SourceStrategyExtensionUrl, StringComparison.Ordinal)) + { + if (!Enum.TryParse(value?.ToString(), ignoreCase: true, out VectorTextSourceStrategy sourceStrategy)) + { + throw new InvalidDefinitionException($"Vector SearchParameter source strategy '{value}' is not supported."); + } + + configuration.SourceStrategy = sourceStrategy; + } + else if (string.Equals(url, VectorSearchParameterConfig.MaxInputTokensExtensionUrl, StringComparison.Ordinal)) + { + if (!int.TryParse(value?.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out int maxInputTokens) || maxInputTokens <= 0) + { + throw new InvalidDefinitionException("Vector SearchParameter maxInputTokens must be greater than zero."); + } + + configuration.MaxInputTokens = maxInputTokens; + } + else if (string.Equals(url, VectorSearchParameterConfig.MinimumScoreExtensionUrl, StringComparison.Ordinal)) + { + if (!decimal.TryParse(value?.ToString(), NumberStyles.Number, CultureInfo.InvariantCulture, out decimal minimumScore) || minimumScore < 0 || minimumScore > 1) + { + throw new InvalidDefinitionException("Vector SearchParameter minimumScore must be between zero and one."); + } + + configuration.MinimumScore = minimumScore; + } + else if (string.Equals(url, VectorSearchParameterConfig.ChunkSizeTokensExtensionUrl, StringComparison.Ordinal)) + { + if (!int.TryParse(value?.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out int chunkSizeTokens) || chunkSizeTokens <= 0) + { + throw new InvalidDefinitionException("Vector SearchParameter chunkSizeTokens must be greater than zero."); + } + + configuration.ChunkSizeTokens = chunkSizeTokens; + } + else if (string.Equals(url, VectorSearchParameterConfig.ChunkOverlapTokensExtensionUrl, StringComparison.Ordinal)) + { + if (!int.TryParse(value?.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out int chunkOverlapTokens) || chunkOverlapTokens < 0) + { + throw new InvalidDefinitionException("Vector SearchParameter chunkOverlapTokens must be non-negative."); + } + + configuration.ChunkOverlapTokens = chunkOverlapTokens; + } + else if (string.Equals(url, VectorSearchParameterConfig.DistanceMetricExtensionUrl, StringComparison.Ordinal)) + { + string distanceMetric = value?.ToString(); + if (!string.Equals(distanceMetric, VectorSearchConfiguration.SupportedDistanceMetric, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidDefinitionException($"Vector SearchParameter distanceMetric must be '{VectorSearchConfiguration.SupportedDistanceMetric}'."); + } + + configuration.DistanceMetric = distanceMetric.ToLowerInvariant(); + } + } + + if (configuration.ChunkSizeTokens.HasValue && + configuration.ChunkOverlapTokens >= configuration.ChunkSizeTokens) + { + throw new InvalidDefinitionException("Vector SearchParameter chunkOverlapTokens must be smaller than chunkSizeTokens."); + } + + return configuration; + } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs index 6b31ee4fa2..d4a32465e2 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs @@ -18,5 +18,6 @@ public enum JobType : int BulkUpdateOrchestrator = 8, ReindexOrchestrator = 9, ReindexProcessing = 10, + VectorSearchSourceRefresh = 11, } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs index 895371eb5f..779b479a10 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs @@ -21,6 +21,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Parameters; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.JobManagement; using Newtonsoft.Json; @@ -57,6 +58,7 @@ public class ReindexProcessingJob : IJob private readonly Func> _fhirDataStoreFactory; private readonly ILogger _logger; private readonly ISearchParameterOperations _searchParameterOperations; + private readonly IVectorSearchIndexer _vectorSearchIndexer; private JobInfo _jobInfo; private ReindexProcessingJobResult _result; @@ -72,7 +74,8 @@ public ReindexProcessingJob( Func> fhirDataStoreFactory, IResourceWrapperFactory resourceWrapperFactory, ISearchParameterOperations searchParameterOperations, - ILogger logger) + ILogger logger, + IVectorSearchIndexer vectorSearchIndexer = null) { EnsureArg.IsNotNull(searchServiceFactory, nameof(searchServiceFactory)); EnsureArg.IsNotNull(fhirDataStoreFactory, nameof(fhirDataStoreFactory)); @@ -85,6 +88,7 @@ public ReindexProcessingJob( _resourceWrapperFactory = resourceWrapperFactory; _searchParameterOperations = searchParameterOperations; _logger = logger; + _vectorSearchIndexer = vectorSearchIndexer; } public static int OomRetryDelayBaseSec { get; set; } = 120; @@ -300,6 +304,11 @@ internal async Task ComputeAndWrite(IReadOnlyList resources, IF _resourceWrapperFactory.Update(resource); } + if (_vectorSearchIndexer != null) + { + await _vectorSearchIndexer.IndexAsync(resources, cancellationToken); + } + await _bulkUpdateRetries.ExecuteAsync(async () => await store.BulkUpdateSearchParameterIndicesAsync(resources, cancellationToken)); } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/VectorSearchSourceRefreshJobDefinition.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/VectorSearchSourceRefreshJobDefinition.cs new file mode 100644 index 0000000000..2e345061ae --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/VectorSearchSourceRefreshJobDefinition.cs @@ -0,0 +1,35 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.JobManagement; + +namespace Microsoft.Health.Fhir.Core.Features.Operations.Reindex +{ + /// + /// Describes a source-resource change that requires dependent vector search indices to be refreshed. + /// + public class VectorSearchSourceRefreshJobDefinition : IJobData + { + /// + /// Gets or sets the job type identifier. + /// + public int TypeId { get; set; } + + /// + /// Gets or sets the source resource type. + /// + public string SourceResourceType { get; set; } + + /// + /// Gets or sets the source resource identifier. + /// + public string SourceResourceId { get; set; } + + /// + /// Gets or sets the source resource version that caused the refresh. + /// + public string SourceResourceVersion { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapper.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapper.cs index 91ab3106b1..1da7be3840 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapper.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapper.cs @@ -9,6 +9,7 @@ using Microsoft.Health.Core; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; using Newtonsoft.Json; @@ -108,6 +109,15 @@ protected ResourceWrapper() [JsonProperty(KnownResourceWrapperProperties.SearchIndices)] public virtual IReadOnlyCollection SearchIndices { get; set; } + [JsonIgnore] + public IReadOnlyCollection VectorSearchIndices { get; private set; } = Array.Empty(); + + /// + /// Gets a value indicating whether vector search indices were evaluated for this wrapper. + /// + [JsonIgnore] + public bool VectorSearchIndicesUpdated { get; private set; } + [JsonProperty(KnownResourceWrapperProperties.LastModifiedClaims)] public IReadOnlyCollection> LastModifiedClaims { get; protected set; } @@ -127,6 +137,12 @@ public virtual void UpdateSearchIndices(IReadOnlyCollection se SearchIndices = searchIndices; } + public void UpdateVectorSearchIndices(IReadOnlyCollection vectorSearchIndices) + { + VectorSearchIndices = EnsureArg.IsNotNull(vectorSearchIndices, nameof(vectorSearchIndices)); + VectorSearchIndicesUpdated = true; + } + internal void UpdateSearchIndices(IReadOnlyCollection searchIndices, string searchParameterHash) { UpdateSearchIndices(searchIndices); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Routing/KnownRoutes.cs b/src/Microsoft.Health.Fhir.Core/Features/Routing/KnownRoutes.cs index fc51a2e60c..210ee29b26 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Routing/KnownRoutes.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Routing/KnownRoutes.cs @@ -110,6 +110,9 @@ internal class KnownRoutes public const string DocRefResourceType = KnownResourceTypes.DocumentReference + "/" + DocRef; public const string DocRefOperationDefinition = OperationDefinition + "/" + OperationsConstants.DocRef; + public const string SemanticSearch = "$semantic-search"; + public const string SemanticSearchPatientById = KnownResourceTypes.Patient + "/" + IdRouteSegment + "/" + SemanticSearch; + public const string Expand = "$expand"; public const string ExpandResourceType = KnownResourceTypes.ValueSet + "/" + Expand; public const string ExpandResourceId = KnownResourceTypes.ValueSet + "/" + IdRouteSegment + "/" + Expand; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/DefaultExpressionVisitor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/DefaultExpressionVisitor.cs index aef568780c..b6482da60c 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/DefaultExpressionVisitor.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/DefaultExpressionVisitor.cs @@ -37,6 +37,8 @@ public virtual TOutput VisitUnion(UnionExpression expression, TContext context) public virtual TOutput VisitSearchParameter(SearchParameterExpression expression, TContext context) => expression.Expression.AcceptVisitor(this, context); + public virtual TOutput VisitVectorSearch(VectorSearchExpression expression, TContext context) => default; + public virtual TOutput VisitBinary(BinaryExpression expression, TContext context) => default; public virtual TOutput VisitChained(ChainedExpression expression, TContext context) => expression.Expression.AcceptVisitor(this, context); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/ExpressionRewriter.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/ExpressionRewriter.cs index 8a0afdf4d2..04d03de5a8 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/ExpressionRewriter.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/ExpressionRewriter.cs @@ -25,6 +25,11 @@ public virtual Expression VisitSearchParameter(SearchParameterExpression express return new SearchParameterExpression(expression.Parameter, visitedExpression); } + public virtual Expression VisitVectorSearch(VectorSearchExpression expression, TContext context) + { + return expression; + } + public virtual Expression VisitBinary(BinaryExpression expression, TContext context) { return expression; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/IExpressionVisitor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/IExpressionVisitor.cs index 76858bb298..7faf7df436 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/IExpressionVisitor.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/IExpressionVisitor.cs @@ -19,6 +19,13 @@ public interface IExpressionVisitor /// The input TOutput VisitSearchParameter(SearchParameterExpression expression, TContext context); + /// + /// Visits the . + /// + /// The expression to visit. + /// The input + TOutput VisitVectorSearch(VectorSearchExpression expression, TContext context); + /// /// Visits the . /// diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/Parsers/SearchParameterExpressionParser.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/Parsers/SearchParameterExpressionParser.cs index 338976f546..ce7ae5b062 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/Parsers/SearchParameterExpressionParser.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/Parsers/SearchParameterExpressionParser.cs @@ -11,6 +11,7 @@ using Hl7.Fhir.Utility; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.ValueSets; @@ -26,10 +27,19 @@ public class SearchParameterExpressionParser : ISearchParameterExpressionParser .Select(e => Tuple.Create(e.GetLiteral(), e)).ToArray(); private readonly Dictionary> _parserDictionary; + private readonly IVectorSearchParameterResolver _vectorSearchParameterResolver; public SearchParameterExpressionParser(IReferenceSearchValueParser referenceSearchValueParser) + : this(referenceSearchValueParser, null) + { + } + + public SearchParameterExpressionParser( + IReferenceSearchValueParser referenceSearchValueParser, + IVectorSearchParameterResolver vectorSearchParameterResolver) { EnsureArg.IsNotNull(referenceSearchValueParser, nameof(referenceSearchValueParser)); + _vectorSearchParameterResolver = vectorSearchParameterResolver; _parserDictionary = new (SearchParamType type, Func parser)[] { @@ -52,6 +62,28 @@ public Expression Parse( EnsureArg.IsNotNull(searchParameter, nameof(searchParameter)); EnsureArg.IsNotNullOrWhiteSpace(value, nameof(value)); + if (searchParameter.Type == SearchParamType.Special && searchParameter.VectorConfig != null) + { + if (modifier != null) + { + throw new InvalidSearchOperationException( + string.Format(CultureInfo.InvariantCulture, Core.Resources.ModifierNotSupported, modifier, searchParameter.Code)); + } + + if (searchParameter.Url == null) + { + throw new InvalidOperationException($"Vector SearchParameter '{searchParameter.Code}' must declare a canonical URL."); + } + + if (_vectorSearchParameterResolver == null) + { + throw new SearchParameterNotSupportedException(searchParameter.Url); + } + + SearchParameterInfo enabledSearchParameter = _vectorSearchParameterResolver.GetSearchParameter(searchParameter.Url); + return new VectorSearchExpression(enabledSearchParameter, value); + } + Expression outputExpression; if (modifier?.SearchModifierCode == SearchModifierCode.Missing) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/VectorSearchExpression.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/VectorSearchExpression.cs new file mode 100644 index 0000000000..15efa00917 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Expressions/VectorSearchExpression.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.Expressions +{ + /// + /// Represents a semantic query over a vector SearchParameter. + /// + public sealed class VectorSearchExpression : SearchParameterExpressionBase + { + /// + /// Initializes a new instance of the class. + /// + /// The vector SearchParameter to query. + /// The text for which a query embedding will be generated. + public VectorSearchExpression(SearchParameterInfo searchParameter, string queryText) + : base(searchParameter) + { + EnsureArg.IsNotNullOrWhiteSpace(queryText, nameof(queryText)); + + QueryText = queryText; + } + + /// + /// Gets the text for which a query embedding will be generated. + /// + public string QueryText { get; } + + /// + public override TOutput AcceptVisitor(IExpressionVisitor visitor, TContext context) + { + EnsureArg.IsNotNull(visitor, nameof(visitor)); + return visitor.VisitVectorSearch(this, context); + } + + /// + public override string ToString() + { + return $"(Vector Param {Parameter.Code})"; + } + + /// + public override void AddValueInsensitiveHashCode(ref HashCode hashCode) + { + hashCode.Add(typeof(VectorSearchExpression)); + hashCode.Add(Parameter); + } + + /// + public override bool ValueInsensitiveEquals(Expression other) + { + return other is VectorSearchExpression vectorExpression && vectorExpression.Parameter.Equals(Parameter); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchParameterInfoExtensions.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchParameterInfoExtensions.cs index 7002c9a80e..47cfdf4e5d 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchParameterInfoExtensions.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchParameterInfoExtensions.cs @@ -52,6 +52,14 @@ internal static string CalculateSearchParameterHash(this IEnumerable s))); } + + if (searchParamInfo.VectorConfig != null) + { + sb.Append(searchParamInfo.VectorConfig.SourceStrategy); + sb.Append(searchParamInfo.VectorConfig.ExtractionPolicy); + sb.Append(searchParamInfo.VectorConfig.MaxInputTokens); + sb.Append(searchParamInfo.VectorConfig.MinimumScore); + } } string hash = sb.ToString().ComputeHash(); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchParameterNames.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchParameterNames.cs index fb2ec53d88..ac62acedd8 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchParameterNames.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchParameterNames.cs @@ -21,6 +21,8 @@ public static class SearchParameterNames public static readonly Uri ResourceTypeUri = new Uri("http://hl7.org/fhir/SearchParameter/Resource-type"); + public const string Score = "_score"; + public static readonly Uri TypeUri = new Uri("http://hl7.org/fhir/SearchParameter/type"); public const string Date = "date"; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchResultEntry.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchResultEntry.cs index 74839a76f7..a507cd53c6 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchResultEntry.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchResultEntry.cs @@ -4,26 +4,51 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using EnsureThat; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.ValueSets; namespace Microsoft.Health.Fhir.Core.Features.Search { public struct SearchResultEntry : IEquatable { - public SearchResultEntry(ResourceWrapper resourceWrapper, SearchEntryMode searchEntryMode = SearchEntryMode.Match) + public SearchResultEntry( + ResourceWrapper resourceWrapper, + SearchEntryMode searchEntryMode = SearchEntryMode.Match, + decimal? score = null, + SemanticSearchEvidence evidence = null, + IReadOnlyList evidenceItems = null) { EnsureArg.IsNotNull(resourceWrapper, nameof(resourceWrapper)); Resource = resourceWrapper; SearchEntryMode = searchEntryMode; + Score = score; + EvidenceItems = evidenceItems ?? (evidence == null ? Array.Empty() : new[] { evidence }); + Evidence = EvidenceItems.Count > 0 ? EvidenceItems[0] : null; } public ResourceWrapper Resource { get; } public SearchEntryMode SearchEntryMode { get; } + /// + /// Gets the normalized semantic relevance score, where higher is more relevant. + /// + public decimal? Score { get; } + + /// + /// Gets the exact passage and provenance supporting this semantic result. + /// + public SemanticSearchEvidence Evidence { get; } + + /// + /// Gets the supporting passages ordered by relevance within this resource. + /// + public IReadOnlyList EvidenceItems { get; } + public static bool operator ==(SearchResultEntry left, SearchResultEntry right) { return left.Equals(right); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/BinaryContentSegment.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/BinaryContentSegment.cs new file mode 100644 index 0000000000..894e1fee8b --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/BinaryContentSegment.cs @@ -0,0 +1,36 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using EnsureThat; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Represents text extracted from one addressable segment of Binary content. + /// + public sealed class BinaryContentSegment + { + /// + /// Initializes a new instance of the class. + /// + /// The extracted text. + /// The optional locator within the Binary data, such as a page number. + public BinaryContentSegment(string text, string sourceLocator = null) + { + Text = EnsureArg.IsNotNull(text, nameof(text)); + SourceLocator = sourceLocator; + } + + /// + /// Gets the extracted text. + /// + public string Text { get; } + + /// + /// Gets the optional locator within the Binary data. + /// + public string SourceLocator { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/DeterministicEmbeddingClient.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/DeterministicEmbeddingClient.cs new file mode 100644 index 0000000000..ff3e2a9854 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/DeterministicEmbeddingClient.cs @@ -0,0 +1,97 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Configs; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// An that produces deterministic, L2-normalized vectors seeded from a hash + /// of the input text. The vectors are reproducible but not semantically meaningful; this client exists so + /// the write and read paths can be exercised offline in tests without calling an external model. + /// + public sealed class DeterministicEmbeddingClient : IEmbeddingClient + { + /// + /// Initializes a new instance of the class. + /// + /// The number of dimensions each embedding should have. + public DeterministicEmbeddingClient(int dimensions = VectorSearchConfiguration.SupportedDimensions) + { + EnsureArg.IsGt(dimensions, 0, nameof(dimensions)); + + Dimensions = dimensions; + } + + /// + public int Dimensions { get; } + + /// + public Task> GenerateEmbeddingsAsync(IReadOnlyList texts, CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(texts, nameof(texts)); + + var embeddings = new List(texts.Count); + + foreach (string text in texts) + { + cancellationToken.ThrowIfCancellationRequested(); + embeddings.Add(Embed(text ?? string.Empty)); + } + + return Task.FromResult>(embeddings); + } + + private float[] Embed(string text) + { + // Derive the vector from a deterministic SHA-256 stream seeded by the text, so identical text always + // yields the identical vector. A hash chain (rather than System.Random) keeps this reproducible and + // avoids a security-sensitive PRNG for what is purely a test fixture. + var vector = new float[Dimensions]; + double sumOfSquares = 0; + + byte[] block = SHA256.HashData(Encoding.UTF8.GetBytes(text)); + int produced = 0; + + while (produced < Dimensions) + { + for (int offset = 0; offset + sizeof(uint) <= block.Length && produced < Dimensions; offset += sizeof(uint)) + { + uint sample = BinaryPrimitives.ReadUInt32LittleEndian(block.AsSpan(offset, sizeof(uint))); + float component = (float)(((double)sample / uint.MaxValue * 2) - 1); + vector[produced++] = component; + sumOfSquares += component * (double)component; + } + + if (produced < Dimensions) + { + // Extend the deterministic byte stream when one hash block is not enough. + block = SHA256.HashData(block); + } + } + + // L2-normalize so cosine distance behaves the way it does for real embeddings. + double magnitude = Math.Sqrt(sumOfSquares); + + if (magnitude > 0) + { + for (int i = 0; i < Dimensions; i++) + { + vector[i] = (float)(vector[i] / magnitude); + } + } + + return vector; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IBinaryContentExtractor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IBinaryContentExtractor.cs new file mode 100644 index 0000000000..b0eefedb56 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IBinaryContentExtractor.cs @@ -0,0 +1,37 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Extracts text from Binary content for vector indexing. + /// + public interface IBinaryContentExtractor + { + /// + /// Gets the normalized MIME types supported by this extractor. + /// + IReadOnlyCollection SupportedContentTypes { get; } + + /// + /// Gets the maximum decoded Binary content length accepted by this extractor. + /// + /// The maximum extracted text length. + /// The maximum decoded content length in bytes. + int GetMaximumContentLength(int maximumTextLength); + + /// + /// Extracts text from decoded Binary data. + /// + /// The decoded Binary data. + /// The original Binary content type. + /// The maximum extracted text length. + /// The ordered extracted text segments when successful. + /// when one or more non-empty text segments were extracted; otherwise . + bool TryExtract(byte[] content, string contentType, int maximumTextLength, out IReadOnlyList segments); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IDocumentReferenceSemanticSearch.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IDocumentReferenceSemanticSearch.cs new file mode 100644 index 0000000000..1d8191741e --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IDocumentReferenceSemanticSearch.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Persistence; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Ranks candidate DocumentReference resources using semantic similarity. + /// + public interface IDocumentReferenceSemanticSearch + { + /// + /// Ranks already-authorized candidate resources for a natural-language query. + /// + Task> SearchAsync( + string query, + IReadOnlyList candidates, + int count, + CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IEmbeddingClient.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IEmbeddingClient.cs new file mode 100644 index 0000000000..de07a0e064 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IEmbeddingClient.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Produces vector embeddings for text by calling an embedding model. Implementations may call an + /// external endpoint (production) or generate vectors locally (tests). + /// + public interface IEmbeddingClient + { + /// + /// Gets the number of dimensions in every embedding this client produces. + /// + int Dimensions { get; } + + /// + /// Produces one embedding per input text. + /// + /// The texts to embed. Must not be null. + /// The cancellation token. + /// One vector per input text, each of length , in the same order as . + Task> GenerateEmbeddingsAsync(IReadOnlyList texts, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IEmbeddingModelRegistry.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IEmbeddingModelRegistry.cs new file mode 100644 index 0000000000..cf5e7668a7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IEmbeddingModelRegistry.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Resolves the database identifier for the configured embedding model. + /// + public interface IEmbeddingModelRegistry + { + /// + /// Gets the database-local identifier for the configured embedding model. + /// + /// A token used to cancel the operation. + /// The embedding model identifier. + Task GetEmbeddingModelIdAsync(CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/ISemanticSearchEvidenceFilter.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/ISemanticSearchEvidenceFilter.cs new file mode 100644 index 0000000000..0ea506226f --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/ISemanticSearchEvidenceFilter.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Removes semantic results whose supporting source resources are not visible to the current request. + /// + public interface ISemanticSearchEvidenceFilter + { + /// + /// Filters semantic results using the authorization context applied by FHIR search. + /// + /// The search result containing semantic evidence. + /// A token to cancel the operation. + /// The filtered search result. + Task FilterAsync(SearchResult searchResult, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/ITextChunker.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/ITextChunker.cs new file mode 100644 index 0000000000..e985826cf2 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/ITextChunker.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Splits document text into overlapping passages so each passage can be embedded on its own. + /// + public interface ITextChunker + { + /// + /// Splits into ordered, overlapping passages. + /// + /// The text to split. Must not be null. + /// The maximum length of each passage. Must be greater than zero. + /// The number of trailing characters each passage shares with the next. Must be at least zero and less than . + /// The ordered passages, or an empty list when is empty. + IReadOnlyList Chunk(string text, int chunkSize, int chunkOverlap); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorResourceReader.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorResourceReader.cs new file mode 100644 index 0000000000..9f402742c7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorResourceReader.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Persistence; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Reads a persisted FHIR resource needed to resolve vector source text. + /// + public interface IVectorResourceReader + { + /// + /// Reads the resource identified by . + /// + /// The resource key. + /// The cancellation token. + /// The resource, or when it does not exist. + Task GetAsync(ResourceKey key, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchIndexer.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchIndexer.cs new file mode 100644 index 0000000000..3e99447461 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchIndexer.cs @@ -0,0 +1,26 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Persistence; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Generates vector search indices from ordinary FHIR SearchParameter extraction results. + /// + public interface IVectorSearchIndexer + { + /// + /// Adds vector search indices to the supplied resource wrappers. + /// + /// The resources to index. + /// A token used to cancel the operation. + /// A task representing the indexing operation. + Task IndexAsync(IReadOnlyCollection resources, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchParameterResolver.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchParameterResolver.cs new file mode 100644 index 0000000000..994e9a582a --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchParameterResolver.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Resolves FHIR SearchParameters configured for vector indexing and querying. + /// + public interface IVectorSearchParameterResolver + { + /// + /// Gets vector SearchParameters applicable to a FHIR resource type. + /// + /// The FHIR resource type. + /// The applicable vector SearchParameters. + IReadOnlyList GetSearchParameters(string resourceType); + + /// + /// Gets vector SearchParameters eligible for indexing a FHIR resource type, including supported definitions awaiting activation. + /// + /// The FHIR resource type. + /// The applicable vector SearchParameters. + IReadOnlyList GetIndexingSearchParameters(string resourceType); + + /// + /// Gets and validates a vector SearchParameter by canonical URI. + /// + /// The SearchParameter canonical URI. + /// The resolved SearchParameter. + SearchParameterInfo GetSearchParameter(Uri canonicalUri); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchQueryProcessor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchQueryProcessor.cs new file mode 100644 index 0000000000..1d40fa6d31 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchQueryProcessor.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Prepares parsed vector search expressions for execution against vector storage. + /// + public interface IVectorSearchQueryProcessor + { + /// + /// Generates the embedding and resolves model provenance for a parsed vector search expression. + /// + /// The parsed FHIR search expression, or when no filters were supplied. + /// A token used to cancel embedding generation and model resolution. + /// The prepared query, or when the expression contains no vector search. + Task PrepareAsync(Expression expression, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchSourceDependencyStore.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchSourceDependencyStore.cs new file mode 100644 index 0000000000..d78ac06c21 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorSearchSourceDependencyStore.cs @@ -0,0 +1,30 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Persistence; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Finds current resources whose vector search indices depend on another resource. + /// + public interface IVectorSearchSourceDependencyStore + { + /// + /// Gets current resource keys with vector search provenance pointing to the specified source. + /// + /// The source resource type. + /// The source resource identifier. + /// A cancellation token. + /// The distinct dependent resource keys. + Task> GetDependentResourceKeysAsync( + string sourceResourceType, + string sourceResourceId, + CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorStore.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorStore.cs new file mode 100644 index 0000000000..009ae2f7b1 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorStore.cs @@ -0,0 +1,61 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Stores per-passage embedding vectors so they can later be ranked by similarity. + /// + public interface IVectorStore + { + /// + /// Stores the embedding vectors for one resource's passages. + /// + /// The id of the FHIR resource type the vectors belong to. + /// The surrogate id of the resource the vectors belong to. + /// The id of the semantic search parameter that produced the vectors. + /// The id of the embedding model that produced the vectors. + /// The per-passage vectors to store. + /// The cancellation token. + /// A task representing the asynchronous operation. + Task StoreAsync( + short resourceTypeId, + long resourceSurrogateId, + short searchParamId, + short embeddingModelId, + IReadOnlyList chunks, + CancellationToken cancellationToken); + + /// + /// Ranks the passages of a pre-filtered candidate set by how close they are to the query vector. + /// The candidate set is the result of the structured filter (patient, encounter, date, and so on), + /// so ranking always runs over records the caller is already allowed to see. + /// + /// The id of the FHIR resource type to rank. + /// The id of the semantic search parameter that produced the vectors. + /// The id of the embedding model whose vectors to rank, so the query and stored vectors share one space. + /// The vector distance metric. + /// The embedding of the search string. + /// The surrogate ids that passed the structured filter. + /// The maximum number of resources to return, ordered by their best passage. + /// The maximum number of passages to return for each selected resource. + /// The cancellation token. + /// The closest passages, ordered from most to least relevant. + Task> SearchAsync( + short resourceTypeId, + short searchParamId, + short embeddingModelId, + string distanceMetric, + IReadOnlyList queryEmbedding, + IReadOnlyList candidateResourceSurrogateIds, + int maxResults, + int evidenceCount, + CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorTextSourceResolver.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorTextSourceResolver.cs new file mode 100644 index 0000000000..6e4c455dce --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/IVectorTextSourceResolver.cs @@ -0,0 +1,35 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Resolves SearchParameter values to text and its FHIR provenance. + /// + public interface IVectorTextSourceResolver + { + /// + /// Resolves extracted values to source text. + /// + /// The resource being vector indexed. + /// The SearchParameter that extracted the values. + /// The extracted SearchParameter values. + /// Resources in the current write batch. + /// The cancellation token. + /// The resolved text sources. + Task> ResolveAsync( + ResourceWrapper owner, + SearchParameterInfo searchParameter, + IReadOnlyList extractedValues, + IReadOnlyCollection writeBatch, + CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PdfBinaryContentExtractor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PdfBinaryContentExtractor.cs new file mode 100644 index 0000000000..31fd7a9540 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PdfBinaryContentExtractor.cs @@ -0,0 +1,124 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using EnsureThat; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using UglyToad.PdfPig; +using UglyToad.PdfPig.Content; +using UglyToad.PdfPig.Core; +using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor; +using UglyToad.PdfPig.Exceptions; +using UglyToad.PdfPig.Fonts; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Extracts page-scoped text from PDF Binary content. + /// + public sealed class PdfBinaryContentExtractor : IBinaryContentExtractor + { + private readonly VectorSearchPdfConfiguration _configuration; + + /// + /// Initializes a new instance of the class. + /// + /// The vector search configuration. + public PdfBinaryContentExtractor(IOptions configuration) + { + _configuration = EnsureArg.IsNotNull(configuration, nameof(configuration)).Value.Indexing.Pdf; + } + + /// + public IReadOnlyCollection SupportedContentTypes { get; } = new[] { "application/pdf" }; + + /// + public int GetMaximumContentLength(int maximumTextLength) + { + return _configuration.MaximumFileSizeBytes; + } + + /// + public bool TryExtract(byte[] content, string contentType, int maximumTextLength, out IReadOnlyList segments) + { + EnsureArg.IsNotNull(content, nameof(content)); + EnsureArg.IsNotNullOrWhiteSpace(contentType, nameof(contentType)); + EnsureArg.IsGt(maximumTextLength, 0, nameof(maximumTextLength)); + + segments = null; + if (content.Length == 0 || content.Length > _configuration.MaximumFileSizeBytes) + { + return false; + } + + try + { + var stopwatch = Stopwatch.StartNew(); + using PdfDocument document = PdfDocument.Open(content); + if (stopwatch.Elapsed > _configuration.ExtractionTimeout) + { + return false; + } + + if (document.NumberOfPages <= 0 || document.NumberOfPages > _configuration.MaximumPageCount) + { + return false; + } + + var extractedSegments = new List(); + int maximumCharacters = Math.Min(maximumTextLength, _configuration.MaximumExtractedCharacters); + int extractedCharacterCount = 0; + + for (int pageNumber = 1; pageNumber <= document.NumberOfPages; pageNumber++) + { + if (stopwatch.Elapsed > _configuration.ExtractionTimeout) + { + return false; + } + + Page page = document.GetPage(pageNumber); + string text = ContentOrderTextExtractor.GetText(page); + if (stopwatch.Elapsed > _configuration.ExtractionTimeout) + { + return false; + } + + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + if (text.Length > maximumCharacters - extractedCharacterCount) + { + return false; + } + + extractedCharacterCount += text.Length; + extractedSegments.Add(new BinaryContentSegment(text, $"page={pageNumber}")); + } + + if (extractedSegments.Count == 0) + { + return false; + } + + segments = extractedSegments; + return true; + } + catch (Exception exception) when ( + exception is PdfDocumentFormatException or + PdfDocumentStackDepthException or + PdfDocumentEncryptedException or + CorruptCompressedDataException or + InvalidFontFormatException) + { + return false; + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PlainTextBinaryContentExtractor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PlainTextBinaryContentExtractor.cs new file mode 100644 index 0000000000..c5ea7ae13e --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PlainTextBinaryContentExtractor.cs @@ -0,0 +1,70 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Extracts strict UTF-8 text from plain-text Binary content. + /// + public sealed class PlainTextBinaryContentExtractor : IBinaryContentExtractor + { + private static readonly UTF8Encoding StrictUtf8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + /// + public IReadOnlyCollection SupportedContentTypes { get; } = new[] { "text/plain" }; + + /// + public int GetMaximumContentLength(int maximumTextLength) + { + return maximumTextLength; + } + + /// + public bool TryExtract(byte[] content, string contentType, int maximumTextLength, out IReadOnlyList segments) + { + EnsureArg.IsNotNull(content, nameof(content)); + EnsureArg.IsNotNullOrWhiteSpace(contentType, nameof(contentType)); + EnsureArg.IsGt(maximumTextLength, 0, nameof(maximumTextLength)); + + segments = null; + if (!IsUtf8(contentType)) + { + return false; + } + + try + { + string extractedText = StrictUtf8.GetString(content); + if (extractedText.Length > maximumTextLength || string.IsNullOrWhiteSpace(extractedText)) + { + return false; + } + + segments = new[] { new BinaryContentSegment(extractedText) }; + return true; + } + catch (DecoderFallbackException) + { + return false; + } + } + + private static bool IsUtf8(string contentType) + { + string charset = contentType + .Split(';', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) + .Skip(1) + .FirstOrDefault(part => part.StartsWith("charset=", StringComparison.OrdinalIgnoreCase)); + + return charset == null || string.Equals(charset.Substring("charset=".Length).Trim('"'), "utf-8", StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PreparedVectorSearchChainLink.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PreparedVectorSearchChainLink.cs new file mode 100644 index 0000000000..2bcc4864d1 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PreparedVectorSearchChainLink.cs @@ -0,0 +1,54 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Describes one relationship traversed from a search root to the resource that owns a vector match. + /// + public sealed class PreparedVectorSearchChainLink + { + /// + /// Initializes a new instance of the class. + /// + /// The resource types that define the reference source. + /// The reference SearchParameter connecting the resources. + /// The resource types targeted by the reference. + /// Whether the relationship is traversed using reverse chaining. + public PreparedVectorSearchChainLink( + IReadOnlyCollection resourceTypes, + SearchParameterInfo referenceSearchParameter, + IReadOnlyCollection targetResourceTypes, + bool reversed) + { + EnsureArg.IsNotNull(resourceTypes, nameof(resourceTypes)); + EnsureArg.IsNotNull(referenceSearchParameter, nameof(referenceSearchParameter)); + EnsureArg.IsNotNull(targetResourceTypes, nameof(targetResourceTypes)); + + ResourceTypes = Array.AsReadOnly(resourceTypes.ToArray()); + ReferenceSearchParameter = referenceSearchParameter; + TargetResourceTypes = Array.AsReadOnly(targetResourceTypes.ToArray()); + Reversed = reversed; + } + + /// Gets the resource types that define the reference source. + public IReadOnlyList ResourceTypes { get; } + + /// Gets the reference SearchParameter connecting the resources. + public SearchParameterInfo ReferenceSearchParameter { get; } + + /// Gets the resource types targeted by the reference. + public IReadOnlyList TargetResourceTypes { get; } + + /// Gets a value indicating whether the relationship is traversed using reverse chaining. + public bool Reversed { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PreparedVectorSearchQuery.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PreparedVectorSearchQuery.cs new file mode 100644 index 0000000000..f616312bba --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/PreparedVectorSearchQuery.cs @@ -0,0 +1,81 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Contains the validated model and embedding inputs required to execute one vector search. + /// + public sealed class PreparedVectorSearchQuery + { + /// + /// Initializes a new instance of the class. + /// + /// The vector SearchParameter being queried. + /// The database-local embedding model identifier. + /// The query embedding. + /// The minimum normalized relevance score required for a chunk to match. + /// The ordered relationships from the search root to the vector-owning resource. + public PreparedVectorSearchQuery( + SearchParameterInfo searchParameter, + short embeddingModelId, + IReadOnlyList embedding, + decimal minimumScore = 0, + IReadOnlyList chainLinks = null) + { + SearchParameter = EnsureArg.IsNotNull(searchParameter, nameof(searchParameter)); + EnsureArg.IsNotNull(embedding, nameof(embedding)); + + if (minimumScore < 0 || minimumScore > 1) + { + throw new ArgumentOutOfRangeException(nameof(minimumScore), minimumScore, "The minimum semantic relevance score must be between zero and one."); + } + + if (embedding.Count != VectorSearchConfiguration.SupportedDimensions) + { + throw new ArgumentException( + $"The query embedding must contain {VectorSearchConfiguration.SupportedDimensions} dimensions.", + nameof(embedding)); + } + + EmbeddingModelId = embeddingModelId; + Embedding = Array.AsReadOnly(embedding.ToArray()); + MinimumScore = minimumScore; + ChainLinks = Array.AsReadOnly((chainLinks ?? Array.Empty()).ToArray()); + } + + /// + /// Gets the vector SearchParameter being queried. + /// + public SearchParameterInfo SearchParameter { get; } + + /// + /// Gets the database-local embedding model identifier. + /// + public short EmbeddingModelId { get; } + + /// + /// Gets an immutable copy of the query embedding. + /// + public IReadOnlyList Embedding { get; } + + /// + /// Gets the minimum normalized relevance score required for a chunk to match. + /// + public decimal MinimumScore { get; } + + /// + /// Gets the ordered relationships from the search root to the vector-owning resource. + /// + public IReadOnlyList ChainLinks { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/SemanticSearchEvidence.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/SemanticSearchEvidence.cs new file mode 100644 index 0000000000..0ef1b8d7ea --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/SemanticSearchEvidence.cs @@ -0,0 +1,193 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Describes the exact source passage that supports one semantic-search result. + /// + public sealed class SemanticSearchEvidence + { + /// + /// Gets the canonical URL of the extension carried on Bundle.entry.search. + /// + public const string ExtensionUrl = "http://microsoft.com/fhir/StructureDefinition/semantic-search-evidence"; + + /// + /// Gets the nested extension URL for the matched passage text. + /// + public const string TextExtensionUrl = "text"; + + /// + /// Gets the nested extension URL for the passage ordinal. + /// + public const string ChunkOrdinalExtensionUrl = "chunkOrdinal"; + + /// + /// Gets the nested extension URL for the one-based relevance rank across evidence on the current response page. + /// + public const string RankExtensionUrl = "rank"; + + /// + /// Gets the nested extension URL for the normalized passage relevance score. + /// + public const string ScoreExtensionUrl = "score"; + + /// + /// Gets the nested extension URL for the vector SearchParameter canonical. + /// + public const string SearchParameterExtensionUrl = "searchParameter"; + + /// + /// Gets the nested extension URL for the FHIR resource containing the source text. + /// + public const string SourceExtensionUrl = "source"; + + /// + /// Gets the nested extension URL for the related resource whose vector produced the root match. + /// + public const string WitnessExtensionUrl = "witness"; + + /// + /// Gets the nested extension URL for the source element path. + /// + public const string SourcePathExtensionUrl = "sourcePath"; + + /// + /// Initializes a new instance of the class. + /// + /// The exact passage text represented by the matched embedding. + /// The zero-based ordinal of the passage within the indexed text. + /// The canonical URL of the SearchParameter that selected the text. + /// The FHIR reference to the resource containing the source text. + /// The path of the source element within the referenced resource. + public SemanticSearchEvidence( + string text, + int chunkOrdinal, + Uri searchParameterCanonical, + string sourceReference, + string sourcePath) + : this(text, chunkOrdinal, score: null, searchParameterCanonical, sourceReference, sourcePath) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The exact passage text represented by the matched embedding. + /// The zero-based ordinal of the passage within the indexed text. + /// The normalized passage relevance score, where higher is more relevant. + /// The canonical URL of the SearchParameter that selected the text. + /// The FHIR reference to the resource containing the source text. + /// The path of the source element within the referenced resource. + /// The optional one-based relevance rank across evidence on the current response page. + /// The optional FHIR reference to the related resource whose vector produced the root match. + public SemanticSearchEvidence( + string text, + int chunkOrdinal, + decimal? score, + Uri searchParameterCanonical, + string sourceReference, + string sourcePath, + int? rank = null, + string witnessReference = null) + { + EnsureArg.IsNotNullOrWhiteSpace(text, nameof(text)); + EnsureArg.IsGte(chunkOrdinal, 0, nameof(chunkOrdinal)); + EnsureArg.IsNotNull(searchParameterCanonical, nameof(searchParameterCanonical)); + EnsureArg.IsNotNullOrWhiteSpace(sourceReference, nameof(sourceReference)); + EnsureArg.IsNotNullOrWhiteSpace(sourcePath, nameof(sourcePath)); + if (witnessReference != null) + { + EnsureArg.IsNotNullOrWhiteSpace(witnessReference, nameof(witnessReference)); + } + + if (score is < 0 or > 1) + { + throw new ArgumentOutOfRangeException(nameof(score), score, "The semantic evidence score must be between 0 and 1."); + } + + if (rank is <= 0) + { + throw new ArgumentOutOfRangeException(nameof(rank), rank, "The semantic evidence rank must be greater than zero."); + } + + if (!searchParameterCanonical.IsAbsoluteUri) + { + throw new ArgumentException("The SearchParameter canonical URL must be absolute.", nameof(searchParameterCanonical)); + } + + Text = text; + ChunkOrdinal = chunkOrdinal; + Rank = rank; + Score = score; + SearchParameterCanonical = searchParameterCanonical; + SourceReference = sourceReference; + SourcePath = sourcePath; + WitnessReference = witnessReference; + } + + /// + /// Gets the exact passage text represented by the matched embedding. + /// + public string Text { get; } + + /// + /// Gets the zero-based ordinal of the passage within the indexed text. + /// + public int ChunkOrdinal { get; } + + /// + /// Gets the optional one-based relevance rank across all evidence attached to resources on the current response page. + /// + public int? Rank { get; } + + /// + /// Gets the normalized passage relevance score, where higher is more relevant. + /// + public decimal? Score { get; } + + /// + /// Gets the canonical URL of the SearchParameter that selected the text. + /// + public Uri SearchParameterCanonical { get; } + + /// + /// Gets the FHIR reference to the resource containing the source text. + /// + public string SourceReference { get; } + + /// + /// Gets the optional FHIR reference to the related resource whose vector produced the root match. + /// + public string WitnessReference { get; } + + /// + /// Gets the path of the source element within the referenced resource. + /// + public string SourcePath { get; } + + /// + /// Creates a copy with the specified page-scoped rank. + /// + /// The one-based rank across evidence on the current response page. + /// A copy of this evidence with the rank assigned. + public SemanticSearchEvidence WithRank(int rank) + { + return new SemanticSearchEvidence( + Text, + ChunkOrdinal, + Score, + SearchParameterCanonical, + SourceReference, + SourcePath, + rank, + WitnessReference); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/SemanticSearchEvidenceFilter.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/SemanticSearchEvidenceFilter.cs new file mode 100644 index 0000000000..1a193214f2 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/SemanticSearchEvidenceFilter.cs @@ -0,0 +1,217 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.ValueSets; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Authorizes external semantic evidence sources through the normal FHIR search pipeline. + /// + public sealed class SemanticSearchEvidenceFilter : ISemanticSearchEvidenceFilter + { + private readonly ISearchService _searchService; + private readonly IDataResourceFilter _dataResourceFilter; + + /// + /// Initializes a new instance of the class. + /// + /// The authorized FHIR search service. + /// The standard resource result filter. + public SemanticSearchEvidenceFilter(ISearchService searchService, IDataResourceFilter dataResourceFilter) + { + _searchService = EnsureArg.IsNotNull(searchService, nameof(searchService)); + _dataResourceFilter = EnsureArg.IsNotNull(dataResourceFilter, nameof(dataResourceFilter)); + } + + /// + public async Task FilterAsync(SearchResult searchResult, CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(searchResult, nameof(searchResult)); + + List results = searchResult.Results.ToList(); + var externalSourcesByResult = new Dictionary>(); + var invalidResults = new HashSet(); + var externalSources = new HashSet(); + + for (int index = 0; index < results.Count; index++) + { + SearchResultEntry result = results[index]; + if (result.SearchEntryMode != SearchEntryMode.Match || !result.Score.HasValue) + { + continue; + } + + if (result.EvidenceItems.Count == 0) + { + invalidResults.Add(index); + continue; + } + + var resultSources = new HashSet(); + foreach (SemanticSearchEvidence evidence in result.EvidenceItems) + { + if (!TryAddExternalReference(result, evidence.SourceReference, resultSources) || + (!string.IsNullOrWhiteSpace(evidence.WitnessReference) && + !TryAddExternalReference(result, evidence.WitnessReference, resultSources))) + { + invalidResults.Add(index); + break; + } + } + + if (!invalidResults.Contains(index) && resultSources.Count > 0) + { + externalSourcesByResult[index] = resultSources.ToList(); + externalSources.UnionWith(resultSources); + } + } + + if (invalidResults.Count == 0 && externalSources.Count == 0) + { + return searchResult; + } + + HashSet authorizedSources = await GetAuthorizedSourcesAsync(externalSources, cancellationToken); + var filteredResults = new List(results.Count); + for (int index = 0; index < results.Count; index++) + { + if (invalidResults.Contains(index) || + (externalSourcesByResult.TryGetValue(index, out IReadOnlyList resultSources) && resultSources.Any(source => !authorizedSources.Contains(source)))) + { + continue; + } + + filteredResults.Add(results[index]); + } + + AssignEvidenceRanks(filteredResults); + var filteredSearchResult = new SearchResult( + filteredResults, + searchResult.ContinuationToken, + searchResult.SortOrder, + searchResult.UnsupportedSearchParameters, + searchResult.SearchIssues, + searchResult.IncludesContinuationToken) + { + MaxResourceSurrogateId = searchResult.MaxResourceSurrogateId, + ReindexResult = searchResult.ReindexResult, + TotalCount = filteredResults.Count == results.Count ? searchResult.TotalCount : null, + }; + + return filteredSearchResult; + } + + private async Task> GetAuthorizedSourcesAsync( + IReadOnlyCollection externalSources, + CancellationToken cancellationToken) + { + var authorizedSources = new HashSet(); + foreach (IGrouping resourceTypeGroup in externalSources.GroupBy(source => source.ResourceType, StringComparer.Ordinal)) + { + string ids = string.Join(",", resourceTypeGroup.Select(source => source.Id).Distinct(StringComparer.Ordinal).OrderBy(id => id, StringComparer.Ordinal)); + var searchParameters = new[] + { + Tuple.Create(KnownQueryParameterNames.Id, ids), + }; + + SearchResult sourceSearchResult; + try + { + sourceSearchResult = await _searchService.SearchAsync(resourceTypeGroup.Key, searchParameters, cancellationToken); + } + catch (Exception exception) when (exception is UnauthorizedFhirActionException or ResourceNotSupportedException) + { + continue; + } + + if (sourceSearchResult == null) + { + continue; + } + + sourceSearchResult = _dataResourceFilter.Filter(sourceSearchResult); + authorizedSources.UnionWith(sourceSearchResult.Results + .Where(result => result.SearchEntryMode == SearchEntryMode.Match) + .Select(result => new SourceIdentity(result.Resource.ResourceTypeName, result.Resource.ResourceId))); + } + + return authorizedSources; + } + + private static bool TryParseSourceReference(string sourceReference, out SourceIdentity source) + { + source = default; + string[] segments = sourceReference?.Split('/') ?? Array.Empty(); + bool validShape = segments.Length == 2 || + (segments.Length == 4 && string.Equals(segments[2], "_history", StringComparison.Ordinal) && !string.IsNullOrWhiteSpace(segments[3])); + if (!validShape || + string.IsNullOrWhiteSpace(segments[0]) || + string.IsNullOrWhiteSpace(segments[1])) + { + return false; + } + + source = new SourceIdentity(segments[0], segments[1]); + return true; + } + + private static bool TryAddExternalReference( + SearchResultEntry result, + string reference, + ISet resultSources) + { + if (!TryParseSourceReference(reference, out SourceIdentity source)) + { + return false; + } + + if (!IsOwnerSource(result, source)) + { + resultSources.Add(source); + } + + return true; + } + + private static bool IsOwnerSource(SearchResultEntry result, SourceIdentity source) + { + return string.Equals(result.Resource.ResourceTypeName, source.ResourceType, StringComparison.Ordinal) && + string.Equals(result.Resource.ResourceId, source.Id, StringComparison.Ordinal); + } + + private static void AssignEvidenceRanks(List results) + { + int[] semanticResultIndexes = results + .Select((result, index) => (result, index)) + .Where(item => item.result.SearchEntryMode == SearchEntryMode.Match && item.result.EvidenceItems.Count > 0) + .Select(item => item.index) + .ToArray(); + IReadOnlyList> rankedEvidence = SemanticSearchEvidenceRanker.AssignRanks( + semanticResultIndexes.Select(index => results[index].EvidenceItems).ToList()); + + for (int index = 0; index < semanticResultIndexes.Length; index++) + { + int resultIndex = semanticResultIndexes[index]; + SearchResultEntry result = results[resultIndex]; + results[resultIndex] = new SearchResultEntry( + result.Resource, + result.SearchEntryMode, + result.Score, + evidenceItems: rankedEvidence[index]); + } + } + + private readonly record struct SourceIdentity(string ResourceType, string Id); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/SemanticSearchEvidenceRanker.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/SemanticSearchEvidenceRanker.cs new file mode 100644 index 0000000000..3060ebbb10 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/SemanticSearchEvidenceRanker.cs @@ -0,0 +1,54 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Linq; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Assigns page-scoped ranks to semantic evidence without changing resource or evidence order. + /// + public static class SemanticSearchEvidenceRanker + { + /// + /// Assigns dense one-based ranks across all evidence attached to resources on the current response page. + /// + /// Evidence grouped in response resource order and relevance order. + /// Evidence in the original grouping and order with ranks assigned. + public static IReadOnlyList> AssignRanks( + IReadOnlyList> evidenceByResource) + { + EnsureArg.IsNotNull(evidenceByResource, nameof(evidenceByResource)); + + int[][] ranks = evidenceByResource + .Select(evidenceItems => new int[evidenceItems.Count]) + .ToArray(); + int rank = 1; + + foreach (var item in evidenceByResource + .SelectMany((evidenceItems, resourceIndex) => evidenceItems.Select((evidence, evidenceIndex) => new + { + Evidence = evidence, + ResourceIndex = resourceIndex, + EvidenceIndex = evidenceIndex, + })) + .OrderByDescending(item => item.Evidence.Score ?? decimal.MinValue) + .ThenBy(item => item.ResourceIndex) + .ThenBy(item => item.EvidenceIndex)) + { + ranks[item.ResourceIndex][item.EvidenceIndex] = rank++; + } + + return evidenceByResource + .Select((evidenceItems, resourceIndex) => + (IReadOnlyList)evidenceItems + .Select((evidence, evidenceIndex) => evidence.WithRank(ranks[resourceIndex][evidenceIndex])) + .ToList()) + .ToList(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/TextChunker.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/TextChunker.cs new file mode 100644 index 0000000000..222f875e95 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/TextChunker.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Splits text into fixed-size, overlapping passages using a sliding character window. The overlap + /// keeps a clinical statement that lands on a boundary from being split across two passages. + /// + public sealed class TextChunker : ITextChunker + { + /// + public IReadOnlyList Chunk(string text, int chunkSize, int chunkOverlap) + { + EnsureArg.IsNotNull(text, nameof(text)); + EnsureArg.IsGt(chunkSize, 0, nameof(chunkSize)); + EnsureArg.IsGte(chunkOverlap, 0, nameof(chunkOverlap)); + EnsureArg.IsLt(chunkOverlap, chunkSize, nameof(chunkOverlap)); + + if (text.Length <= chunkSize) + { + return text.Length == 0 ? Array.Empty() : new[] { text }; + } + + int step = chunkSize - chunkOverlap; + var chunks = new List(); + + for (int start = 0; start < text.Length; start += step) + { + int length = Math.Min(chunkSize, text.Length - start); + chunks.Add(text.Substring(start, length)); + + if (start + length == text.Length) + { + break; + } + } + + return chunks; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchChunk.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchChunk.cs new file mode 100644 index 0000000000..d1900880d5 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchChunk.cs @@ -0,0 +1,92 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// One passage's embedding vector, identified by its ordinal within the source document. + /// + public sealed class VectorSearchChunk + { + /// + /// Initializes a new instance of the class. + /// + /// The zero-based ordinal of this passage within the source document. + /// The exact passage text represented by the embedding. + /// The hash of the exact passage text. + /// The embedding vector for this passage. + /// The source resource type. + /// The source resource id. + /// The source resource version. + /// The source element path. + public VectorSearchChunk( + int chunkOrdinal, + string chunkText, + IReadOnlyList sourceTextHash, + IReadOnlyList embedding, + string sourceResourceType = null, + string sourceResourceId = null, + string sourceResourceVersion = null, + string sourcePath = null) + { + EnsureArg.IsGte(chunkOrdinal, 0, nameof(chunkOrdinal)); + EnsureArg.IsNotNull(chunkText, nameof(chunkText)); + EnsureArg.IsNotNull(sourceTextHash, nameof(sourceTextHash)); + EnsureArg.IsNotNull(embedding, nameof(embedding)); + + ChunkOrdinal = chunkOrdinal; + ChunkText = chunkText; + SourceTextHash = sourceTextHash; + Embedding = embedding; + SourceResourceType = sourceResourceType; + SourceResourceId = sourceResourceId; + SourceResourceVersion = sourceResourceVersion; + SourcePath = sourcePath; + } + + /// + /// Gets the zero-based ordinal of this passage within the source document. + /// + public int ChunkOrdinal { get; } + + /// + /// Gets the exact passage text represented by the embedding. + /// + public string ChunkText { get; } + + /// + /// Gets the hash of the exact passage text, used to skip re-embedding unchanged text. + /// + public IReadOnlyList SourceTextHash { get; } + + /// + /// Gets the embedding vector for this passage. + /// + public IReadOnlyList Embedding { get; } + + /// + /// Gets the type of the FHIR resource containing the source text. + /// + public string SourceResourceType { get; } + + /// + /// Gets the id of the FHIR resource containing the source text. + /// + public string SourceResourceId { get; } + + /// + /// Gets the version of the FHIR resource containing the source text. + /// + public string SourceResourceVersion { get; } + + /// + /// Gets the path of the source text within the FHIR resource. + /// + public string SourcePath { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchHit.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchHit.cs new file mode 100644 index 0000000000..f1702cfb18 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchHit.cs @@ -0,0 +1,89 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using EnsureThat; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Represents one ranked vector-store hit before FHIR source provenance is attached. + /// + public sealed class VectorSearchHit + { + /// + /// Initializes a new instance of the class. + /// + /// The surrogate id of the resource that matched. + /// The zero-based ordinal of the matched passage. + /// The exact text represented by the matched embedding. + /// The normalized relevance score, where higher is more relevant. + /// The storage resource type id containing the source text. + /// The id of the resource containing the source text. + /// The version of the resource containing the source text. + /// The path of the source text within its resource. + public VectorSearchHit( + long resourceSurrogateId, + int chunkOrdinal, + string chunkText, + float score, + short? sourceResourceTypeId = null, + string sourceResourceId = null, + string sourceResourceVersion = null, + string sourcePath = null) + { + EnsureArg.IsGte(chunkOrdinal, 0, nameof(chunkOrdinal)); + EnsureArg.IsNotNullOrWhiteSpace(chunkText, nameof(chunkText)); + + ResourceSurrogateId = resourceSurrogateId; + ChunkOrdinal = chunkOrdinal; + ChunkText = chunkText; + Score = score; + SourceResourceTypeId = sourceResourceTypeId; + SourceResourceId = sourceResourceId; + SourceResourceVersion = sourceResourceVersion; + SourcePath = sourcePath; + } + + /// + /// Gets the surrogate id of the resource that matched. + /// + public long ResourceSurrogateId { get; } + + /// + /// Gets the zero-based ordinal of the matched passage. + /// + public int ChunkOrdinal { get; } + + /// + /// Gets the exact text represented by the matched embedding. + /// + public string ChunkText { get; } + + /// + /// Gets the normalized relevance score, where higher is more relevant. + /// + public float Score { get; } + + /// + /// Gets the storage resource type id containing the source text. + /// + public short? SourceResourceTypeId { get; } + + /// + /// Gets the id of the resource containing the source text. + /// + public string SourceResourceId { get; } + + /// + /// Gets the version of the resource containing the source text. + /// + public string SourceResourceVersion { get; } + + /// + /// Gets the path of the source text within its resource. + /// + public string SourcePath { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchIndexEntry.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchIndexEntry.cs new file mode 100644 index 0000000000..ecd3e7eb4e --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchIndexEntry.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Contains the vector passages generated for one FHIR SearchParameter on one resource. + /// + public sealed class VectorSearchIndexEntry + { + /// + /// Initializes a new instance of the class. + /// + /// The FHIR SearchParameter that extracted the source values. + /// The database-local embedding model identifier. + /// The ordered embedded passages. + public VectorSearchIndexEntry( + SearchParameterInfo searchParameter, + short embeddingModelId, + IReadOnlyList chunks) + { + SearchParameter = EnsureArg.IsNotNull(searchParameter, nameof(searchParameter)); + EmbeddingModelId = embeddingModelId; + Chunks = EnsureArg.IsNotNull(chunks, nameof(chunks)); + } + + /// + /// Gets the FHIR SearchParameter that extracted the source values. + /// + public SearchParameterInfo SearchParameter { get; } + + /// + /// Gets the database-local embedding model identifier. + /// + public short EmbeddingModelId { get; } + + /// + /// Gets the ordered embedded passages. + /// + public IReadOnlyList Chunks { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchIndexer.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchIndexer.cs new file mode 100644 index 0000000000..f10dd16f3a --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchIndexer.cs @@ -0,0 +1,234 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Generates embeddings from values extracted by enabled FHIR SearchParameters. + /// + public sealed class VectorSearchIndexer : IVectorSearchIndexer + { + private const string ConcatenatedValueSeparator = "\n"; + + private readonly IVectorSearchParameterResolver _searchParameterResolver; + private readonly ITextChunker _textChunker; + private readonly IEmbeddingClient _embeddingClient; + private readonly IEmbeddingModelRegistry _embeddingModelRegistry; + private readonly IVectorTextSourceResolver _textSourceResolver; + private readonly VectorSearchIndexingConfiguration _configuration; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + public VectorSearchIndexer( + IVectorSearchParameterResolver searchParameterResolver, + ITextChunker textChunker, + IEmbeddingClient embeddingClient, + IEmbeddingModelRegistry embeddingModelRegistry, + IVectorTextSourceResolver textSourceResolver, + IOptions configuration, + ILogger logger) + { + _searchParameterResolver = EnsureArg.IsNotNull(searchParameterResolver, nameof(searchParameterResolver)); + _textChunker = EnsureArg.IsNotNull(textChunker, nameof(textChunker)); + _embeddingClient = EnsureArg.IsNotNull(embeddingClient, nameof(embeddingClient)); + _embeddingModelRegistry = EnsureArg.IsNotNull(embeddingModelRegistry, nameof(embeddingModelRegistry)); + _textSourceResolver = EnsureArg.IsNotNull(textSourceResolver, nameof(textSourceResolver)); + _configuration = EnsureArg.IsNotNull(configuration, nameof(configuration)).Value.Indexing; + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + } + + /// + public async Task IndexAsync(IReadOnlyCollection resources, CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(resources, nameof(resources)); + + var entriesByResource = resources.ToDictionary(resource => resource, _ => new List()); + var pendingIndices = new List(); + var passages = new List(); + + foreach (ResourceWrapper resource in resources) + { + resource.UpdateVectorSearchIndices(Array.Empty()); + + if (resource.IsDeleted || resource.IsHistory) + { + continue; + } + + foreach (SearchParameterInfo searchParameter in _searchParameterResolver.GetIndexingSearchParameters(resource.ResourceTypeName)) + { + IReadOnlyList extractedValues = GetExtractedValues(resource, searchParameter); + IReadOnlyList resolvedSources = await _textSourceResolver.ResolveAsync( + resource, + searchParameter, + extractedValues, + resources, + cancellationToken); + IReadOnlyList sourceTexts = ApplyExtractionPolicy(searchParameter.VectorConfig.ExtractionPolicy, resolvedSources); + var chunks = new List(); + int configuredChunkSize = searchParameter.VectorConfig.ChunkSizeTokens ?? _configuration.ChunkSizeTokens; + int configuredChunkOverlap = searchParameter.VectorConfig.ChunkOverlapTokens ?? _configuration.ChunkOverlapTokens; + int chunkSize = Math.Min(configuredChunkSize, searchParameter.VectorConfig.MaxInputTokens); + int chunkOverlap = Math.Min(configuredChunkOverlap, chunkSize - 1); + + foreach (VectorTextSource sourceText in sourceTexts) + { + chunks.AddRange(_textChunker + .Chunk(sourceText.Text, chunkSize, chunkOverlap) + .Select(text => new VectorTextSource(text, sourceText.ResourceType, sourceText.ResourceId, sourceText.ResourceVersion, sourceText.Path))); + } + + if (chunks.Count == 0) + { + continue; + } + + if (chunks.Count > short.MaxValue) + { + throw new InvalidOperationException($"Vector SearchParameter '{searchParameter.Url}' generated more than {short.MaxValue} passages for one resource."); + } + + pendingIndices.Add(new PendingVectorIndex(resource, searchParameter, passages.Count, chunks.Count)); + passages.AddRange(chunks); + } + } + + if (pendingIndices.Count == 0) + { + _logger.LogInformation("Vector indexing found no text to embed across {ResourceCount} resource(s); embedding endpoint not invoked.", resources.Count); + return; + } + + _logger.LogInformation("Vector indexing invoking embedding endpoint for {PassageCount} passage(s) across {IndexCount} search-parameter target(s).", passages.Count, pendingIndices.Count); + IReadOnlyList embeddings = await _embeddingClient.GenerateEmbeddingsAsync(passages.Select(passage => passage.Text).ToList(), cancellationToken); + if (embeddings.Count != passages.Count) + { + throw new InvalidOperationException("The embedding service returned a different number of vectors than passages."); + } + + short embeddingModelId = await _embeddingModelRegistry.GetEmbeddingModelIdAsync(cancellationToken); + foreach (PendingVectorIndex pendingIndex in pendingIndices) + { + var chunks = new List(pendingIndex.PassageCount); + for (int chunkOrdinal = 0; chunkOrdinal < pendingIndex.PassageCount; chunkOrdinal++) + { + int passageIndex = pendingIndex.FirstPassageIndex + chunkOrdinal; + float[] embedding = embeddings[passageIndex]; + if (embedding.Length != _embeddingClient.Dimensions) + { + throw new InvalidOperationException($"The embedding service returned a vector with {embedding.Length} dimensions; expected {_embeddingClient.Dimensions}."); + } + + VectorTextSource passage = passages[passageIndex]; + byte[] sourceTextHash = SHA256.HashData(Encoding.UTF8.GetBytes(passage.Text)); + chunks.Add(new VectorSearchChunk( + chunkOrdinal, + passage.Text, + sourceTextHash, + embedding, + passage.ResourceType, + passage.ResourceId, + passage.ResourceVersion, + passage.Path)); + } + + entriesByResource[pendingIndex.Resource].Add( + new VectorSearchIndexEntry(pendingIndex.SearchParameter, embeddingModelId, chunks)); + } + + foreach (KeyValuePair> resourceEntries in entriesByResource) + { + resourceEntries.Key.UpdateVectorSearchIndices(resourceEntries.Value); + } + } + + private static List GetExtractedValues(ResourceWrapper resource, SearchParameterInfo searchParameter) + { + var values = new List(); + foreach (SearchIndexEntry searchIndex in resource.SearchIndices ?? Array.Empty()) + { + if (searchIndex.SearchParameter.Url != searchParameter.Url) + { + continue; + } + + if (searchIndex.Value is not StringSearchValue stringValue) + { + throw new InvalidOperationException($"Vector SearchParameter '{searchParameter.Url}' must extract string values."); + } + + values.Add(stringValue.String); + } + + return values; + } + + private static IReadOnlyList ApplyExtractionPolicy( + VectorTextExtractionPolicy extractionPolicy, + IReadOnlyList extractedValues) + { + if (extractedValues.Count == 0) + { + return Array.Empty(); + } + + return extractionPolicy switch + { + VectorTextExtractionPolicy.FirstValue => new[] { extractedValues[0] }, + VectorTextExtractionPolicy.Concatenate => extractedValues + .GroupBy(value => (value.ResourceType, value.ResourceId, value.ResourceVersion, value.Path)) + .Select(group => new VectorTextSource( + string.Join(ConcatenatedValueSeparator, group.Select(value => value.Text)), + group.Key.ResourceType, + group.Key.ResourceId, + group.Key.ResourceVersion, + group.Key.Path)) + .ToList(), + VectorTextExtractionPolicy.PerValueRow => extractedValues, + _ => throw new InvalidOperationException($"Unsupported vector text extraction policy '{extractionPolicy}'."), + }; + } + + private sealed class PendingVectorIndex + { + public PendingVectorIndex( + ResourceWrapper resource, + SearchParameterInfo searchParameter, + int firstPassageIndex, + int passageCount) + { + Resource = resource; + SearchParameter = searchParameter; + FirstPassageIndex = firstPassageIndex; + PassageCount = passageCount; + } + + public ResourceWrapper Resource { get; } + + public SearchParameterInfo SearchParameter { get; } + + public int FirstPassageIndex { get; } + + public int PassageCount { get; } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchParameterResolver.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchParameterResolver.cs new file mode 100644 index 0000000000..07de587ce7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchParameterResolver.cs @@ -0,0 +1,124 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.ValueSets; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Resolves active vector definitions through the server's FHIR SearchParameter registry. + /// + public sealed class VectorSearchParameterResolver : IVectorSearchParameterResolver + { + private readonly ISearchParameterDefinitionManager _searchParameterDefinitionManager; + + /// + /// Initializes a new instance of the class. + /// + /// The FHIR SearchParameter definition manager. + public VectorSearchParameterResolver(ISearchParameterDefinitionManager searchParameterDefinitionManager) + { + _searchParameterDefinitionManager = EnsureArg.IsNotNull(searchParameterDefinitionManager, nameof(searchParameterDefinitionManager)); + } + + /// + public IReadOnlyList GetSearchParameters(string resourceType) + { + EnsureArg.IsNotNullOrWhiteSpace(resourceType, nameof(resourceType)); + + return _searchParameterDefinitionManager + .GetSearchParameters(resourceType) + .Where(searchParameter => TryValidate(searchParameter, requireSearchable: true, out _)) + .OrderBy(searchParameter => searchParameter.Url.OriginalString, StringComparer.Ordinal) + .ToList(); + } + + /// + public IReadOnlyList GetIndexingSearchParameters(string resourceType) + { + EnsureArg.IsNotNullOrWhiteSpace(resourceType, nameof(resourceType)); + + return _searchParameterDefinitionManager + .GetSearchParameters(resourceType) + .Where(searchParameter => TryValidate(searchParameter, requireSearchable: false, out _)) + .OrderBy(searchParameter => searchParameter.Url.OriginalString, StringComparer.Ordinal) + .ToList(); + } + + /// + public SearchParameterInfo GetSearchParameter(Uri canonicalUri) + { + EnsureArg.IsNotNull(canonicalUri, nameof(canonicalUri)); + + if (!_searchParameterDefinitionManager.TryGetSearchParameter(canonicalUri.OriginalString, excludePendingDelete: true, out SearchParameterInfo searchParameter)) + { + throw new SearchParameterNotSupportedException(canonicalUri); + } + + Validate(searchParameter); + return searchParameter; + } + + private static void Validate(SearchParameterInfo searchParameter) + { + if (!TryValidate(searchParameter, requireSearchable: true, out string errorMessage)) + { + throw new InvalidOperationException(errorMessage); + } + } + + private static bool TryValidate(SearchParameterInfo searchParameter, bool requireSearchable, out string errorMessage) + { + if (searchParameter.Type != SearchParamType.Special) + { + errorMessage = $"Vector SearchParameter '{searchParameter.Url}' must use FHIR type 'special'."; + return false; + } + + if (!string.Equals(searchParameter.DefinitionStatus, "active", StringComparison.OrdinalIgnoreCase)) + { + errorMessage = $"Vector SearchParameter '{searchParameter.Url}' must have FHIR publication status 'active'."; + return false; + } + + if (searchParameter.BaseResourceTypes == null || searchParameter.BaseResourceTypes.Count == 0) + { + errorMessage = $"Vector SearchParameter '{searchParameter.Url}' must declare at least one FHIR base resource type."; + return false; + } + + if (string.IsNullOrWhiteSpace(searchParameter.Expression)) + { + errorMessage = $"Vector SearchParameter '{searchParameter.Url}' must declare an expression."; + return false; + } + + if (searchParameter.VectorConfig == null) + { + errorMessage = $"Vector SearchParameter '{searchParameter.Url}' must declare the '{VectorSearchParameterConfig.ExtensionUrl}' extension."; + return false; + } + + bool isEligibleForIndexing = searchParameter.IsSearchable || searchParameter.SearchParameterStatus == SearchParameterStatus.Supported; + if (!searchParameter.IsSupported || (requireSearchable ? !searchParameter.IsSearchable : !isEligibleForIndexing)) + { + errorMessage = requireSearchable + ? $"Vector SearchParameter '{searchParameter.Url}' must be supported and searchable." + : $"Vector SearchParameter '{searchParameter.Url}' must be enabled or awaiting activation in the supported state."; + return false; + } + + errorMessage = null; + return true; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchQueryProcessor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchQueryProcessor.cs new file mode 100644 index 0000000000..6c2841d907 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchQueryProcessor.cs @@ -0,0 +1,133 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Generates query embeddings inline for parsed vector search expressions. + /// + public sealed class VectorSearchQueryProcessor : IVectorSearchQueryProcessor + { + private readonly IEmbeddingClient _embeddingClient; + private readonly IEmbeddingModelRegistry _embeddingModelRegistry; + + /// + /// Initializes a new instance of the class. + /// + /// The configured embedding client. + /// The embedding model registry. + public VectorSearchQueryProcessor( + IEmbeddingClient embeddingClient, + IEmbeddingModelRegistry embeddingModelRegistry) + { + _embeddingClient = EnsureArg.IsNotNull(embeddingClient, nameof(embeddingClient)); + _embeddingModelRegistry = EnsureArg.IsNotNull(embeddingModelRegistry, nameof(embeddingModelRegistry)); + } + + /// + public async Task PrepareAsync(Expression expression, CancellationToken cancellationToken) + { + if (expression == null) + { + return null; + } + + var collector = new VectorSearchExpressionCollector(); + expression.AcceptVisitor(collector, context: null); + + if (collector.Expressions.Count == 0) + { + return null; + } + + if (collector.Expressions.Count > 1) + { + throw new InvalidSearchOperationException("Only one vector SearchParameter may be specified per search."); + } + + if (_embeddingClient.Dimensions != VectorSearchConfiguration.SupportedDimensions) + { + throw new InvalidOperationException( + $"The embedding client produces {_embeddingClient.Dimensions} dimensions; expected {VectorSearchConfiguration.SupportedDimensions}."); + } + + CollectedVectorSearchExpression collectedExpression = collector.Expressions[0]; + VectorSearchExpression vectorExpression = collectedExpression.Expression; + if (collectedExpression.ChainLinks.Count > 1) + { + throw new InvalidSearchOperationException("Semantic search currently supports one chain relationship."); + } + + IReadOnlyList embeddings = await _embeddingClient.GenerateEmbeddingsAsync( + new[] { vectorExpression.QueryText }, + cancellationToken); + + if (embeddings.Count != 1) + { + throw new InvalidOperationException("The embedding service must return exactly one vector for a semantic query."); + } + + float[] embedding = embeddings[0]; + if (embedding == null || embedding.Length != _embeddingClient.Dimensions) + { + int actualDimensions = embedding?.Length ?? 0; + throw new InvalidOperationException( + $"The embedding service returned a vector with {actualDimensions} dimensions; expected {_embeddingClient.Dimensions}."); + } + + short embeddingModelId = await _embeddingModelRegistry.GetEmbeddingModelIdAsync(cancellationToken); + return new PreparedVectorSearchQuery( + vectorExpression.Parameter, + embeddingModelId, + embedding, + vectorExpression.Parameter.VectorConfig.MinimumScore, + collectedExpression.ChainLinks); + } + + private sealed class VectorSearchExpressionCollector : DefaultExpressionVisitor + { + private readonly List _currentChainLinks = new List(); + + public List Expressions { get; } = new List(); + + public override object VisitChained(ChainedExpression expression, object context) + { + _currentChainLinks.Add(new PreparedVectorSearchChainLink( + expression.ResourceTypes, + expression.ReferenceSearchParameter, + expression.TargetResourceTypes, + expression.Reversed)); + + try + { + return expression.Expression.AcceptVisitor(this, context); + } + finally + { + _currentChainLinks.RemoveAt(_currentChainLinks.Count - 1); + } + } + + public override object VisitVectorSearch(VectorSearchExpression expression, object context) + { + Expressions.Add(new CollectedVectorSearchExpression(expression, _currentChainLinks.ToArray())); + return null; + } + } + + private sealed record CollectedVectorSearchExpression( + VectorSearchExpression Expression, + IReadOnlyList ChainLinks); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchResult.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchResult.cs new file mode 100644 index 0000000000..831a3213a4 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorSearchResult.cs @@ -0,0 +1,73 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// One ranked FHIR semantic-search result and the evidence passage that supports it. + /// + public sealed class VectorSearchResult + { + /// + /// Initializes a new instance of the class. + /// + /// The FHIR resource type that matched. + /// The surrogate id of the resource that matched. + /// The relevance score from 0 (unrelated) to 1 (identical), where higher is more relevant. + /// The exact passage and FHIR source provenance supporting the result. + public VectorSearchResult(string resourceTypeName, long resourceSurrogateId, float score, SemanticSearchEvidence evidence) + : this(resourceTypeName, resourceSurrogateId, score, new[] { evidence }) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The FHIR resource type that matched. + /// The surrogate id of the resource that matched. + /// The relevance score from 0 (unrelated) to 1 (identical), where higher is more relevant. + /// The supporting passages ordered by relevance within this resource. + public VectorSearchResult(string resourceTypeName, long resourceSurrogateId, float score, IReadOnlyList evidenceItems) + { + EnsureArg.IsNotNullOrWhiteSpace(resourceTypeName, nameof(resourceTypeName)); + EnsureArg.IsNotNull(evidenceItems, nameof(evidenceItems)); + EnsureArg.HasItems(evidenceItems, nameof(evidenceItems)); + + ResourceTypeName = resourceTypeName; + ResourceSurrogateId = resourceSurrogateId; + Score = score; + EvidenceItems = evidenceItems; + Evidence = evidenceItems[0]; + } + + /// + /// Gets the FHIR resource type that matched. + /// + public string ResourceTypeName { get; } + + /// + /// Gets the surrogate id of the resource that matched. + /// + public long ResourceSurrogateId { get; } + + /// + /// Gets the exact passage and FHIR source provenance supporting the result. + /// + public SemanticSearchEvidence Evidence { get; } + + /// + /// Gets the supporting passages ordered by relevance within this resource. + /// + public IReadOnlyList EvidenceItems { get; } + + /// + /// Gets the relevance score from 0 (unrelated) to 1 (identical), where higher is more relevant. + /// + public float Score { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorTextSource.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorTextSource.cs new file mode 100644 index 0000000000..09fc16f905 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorTextSource.cs @@ -0,0 +1,57 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using EnsureThat; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Text selected for vector indexing together with its FHIR provenance. + /// + public sealed class VectorTextSource + { + /// + /// Initializes a new instance of the class. + /// + /// The source text. + /// The source resource type. + /// The source resource id. + /// The source resource version. + /// The source element path. + public VectorTextSource(string text, string resourceType, string resourceId, string resourceVersion, string path) + { + Text = EnsureArg.IsNotNull(text, nameof(text)); + ResourceType = EnsureArg.IsNotNullOrWhiteSpace(resourceType, nameof(resourceType)); + ResourceId = EnsureArg.IsNotNullOrWhiteSpace(resourceId, nameof(resourceId)); + ResourceVersion = resourceVersion; + Path = EnsureArg.IsNotNullOrWhiteSpace(path, nameof(path)); + } + + /// + /// Gets the source text. + /// + public string Text { get; } + + /// + /// Gets the source resource type. + /// + public string ResourceType { get; } + + /// + /// Gets the source resource id. + /// + public string ResourceId { get; } + + /// + /// Gets the source resource version. + /// + public string ResourceVersion { get; } + + /// + /// Gets the source element path. + /// + public string Path { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorTextSourceResolver.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorTextSourceResolver.cs new file mode 100644 index 0000000000..0833b583a4 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SemanticSearch/VectorTextSourceResolver.cs @@ -0,0 +1,194 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Hl7.Fhir.ElementModel; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Resolves direct text and local Binary references selected by vector SearchParameters. + /// + public sealed class VectorTextSourceResolver : IVectorTextSourceResolver + { + private const string BinaryResourceType = "Binary"; + private const string BinaryDataPath = "Binary.data"; + private const int MaximumUtf8BytesPerToken = 4; + + private readonly IVectorResourceReader _resourceReader; + private readonly IResourceDeserializer _resourceDeserializer; + private readonly Dictionary _binaryContentExtractors; + + /// + /// Initializes a new instance of the class. + /// + /// The persisted resource reader. + /// The FHIR resource deserializer. + /// The registered Binary content extractors. + public VectorTextSourceResolver( + IVectorResourceReader resourceReader, + IResourceDeserializer resourceDeserializer, + IEnumerable binaryContentExtractors) + { + _resourceReader = EnsureArg.IsNotNull(resourceReader, nameof(resourceReader)); + _resourceDeserializer = EnsureArg.IsNotNull(resourceDeserializer, nameof(resourceDeserializer)); + _binaryContentExtractors = EnsureArg.IsNotNull(binaryContentExtractors, nameof(binaryContentExtractors)) + .SelectMany(extractor => extractor.SupportedContentTypes.Select(contentType => (ContentType: contentType, Extractor: extractor))) + .ToDictionary(item => item.ContentType, item => item.Extractor, StringComparer.OrdinalIgnoreCase); + } + + /// + public async Task> ResolveAsync( + ResourceWrapper owner, + SearchParameterInfo searchParameter, + IReadOnlyList extractedValues, + IReadOnlyCollection writeBatch, + CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(owner, nameof(owner)); + EnsureArg.IsNotNull(searchParameter, nameof(searchParameter)); + EnsureArg.IsNotNull(extractedValues, nameof(extractedValues)); + EnsureArg.IsNotNull(writeBatch, nameof(writeBatch)); + + if (searchParameter.VectorConfig.SourceStrategy == VectorTextSourceStrategy.DirectText) + { + return extractedValues + .Select(value => new VectorTextSource(value, owner.ResourceTypeName, owner.ResourceId, owner.Version, searchParameter.Expression)) + .ToList(); + } + + if (searchParameter.VectorConfig.SourceStrategy != VectorTextSourceStrategy.LocalBinaryReference) + { + throw new InvalidOperationException($"Unsupported vector text source strategy '{searchParameter.VectorConfig.SourceStrategy}'."); + } + + var sources = new List(); + foreach (string extractedValue in extractedValues) + { + if (!TryParseBinaryReference(extractedValue, out string binaryId)) + { + continue; + } + + ResourceWrapper binary = writeBatch + .Reverse() + .FirstOrDefault(resource => + string.Equals(resource.ResourceTypeName, BinaryResourceType, StringComparison.Ordinal) && + string.Equals(resource.ResourceId, binaryId, StringComparison.Ordinal)); + + if (binary == null) + { + binary = await _resourceReader.GetAsync(new ResourceKey(BinaryResourceType, binaryId), cancellationToken); + } + + if (binary == null || binary.IsDeleted || binary.IsHistory || !TryDecodeBinary(binary, searchParameter.VectorConfig.MaxInputTokens, out IReadOnlyList segments)) + { + continue; + } + + sources.AddRange(segments.Select(segment => new VectorTextSource( + segment.Text, + BinaryResourceType, + binary.ResourceId, + binary.Version, + GetSourcePath(segment.SourceLocator)))); + } + + return sources; + } + + private static bool TryParseBinaryReference(string value, out string binaryId) + { + binaryId = null; + if (string.IsNullOrWhiteSpace(value) || Uri.TryCreate(value, UriKind.Absolute, out _)) + { + return false; + } + + string[] segments = value.Split('/'); + if (segments.Length != 2 || + !string.Equals(segments[0], BinaryResourceType, StringComparison.Ordinal) || + string.IsNullOrWhiteSpace(segments[1]) || + segments[1].Contains('?', StringComparison.Ordinal) || + segments[1].Contains('#', StringComparison.Ordinal)) + { + return false; + } + + binaryId = segments[1]; + return true; + } + + private bool TryDecodeBinary(ResourceWrapper binary, int maxInputTokens, out IReadOnlyList segments) + { + segments = null; + + try + { + ResourceElement resource = _resourceDeserializer.Deserialize(binary); + if (!string.Equals(resource.InstanceType, BinaryResourceType, StringComparison.Ordinal)) + { + return false; + } + + string contentType = resource.Instance.Children("contentType").SingleOrDefault()?.Value?.ToString(); + string normalizedContentType = contentType?.Split(';', 2, StringSplitOptions.TrimEntries)[0]; + if (string.IsNullOrWhiteSpace(normalizedContentType) || !_binaryContentExtractors.TryGetValue(normalizedContentType, out IBinaryContentExtractor extractor)) + { + return false; + } + + object data = resource.Instance.Children("data").SingleOrDefault()?.Value; + int maximumTextLength = (int)Math.Min((long)maxInputTokens * MaximumUtf8BytesPerToken, int.MaxValue); + int maximumBytes = extractor.GetMaximumContentLength(maximumTextLength); + if (!TryGetBytes(data, maximumBytes, out byte[] bytes)) + { + return false; + } + + return extractor.TryExtract(bytes, contentType, maximumTextLength, out segments) && segments?.Count > 0; + } + catch (Exception exception) when (exception is FormatException or InvalidOperationException) + { + return false; + } + } + + private static string GetSourcePath(string sourceLocator) + { + return string.IsNullOrWhiteSpace(sourceLocator) ? BinaryDataPath : $"{BinaryDataPath}#{sourceLocator}"; + } + + private static bool TryGetBytes(object data, int maximumBytes, out byte[] bytes) + { + bytes = null; + if (data is byte[] binaryData) + { + if (binaryData.Length > maximumBytes) + { + return false; + } + + bytes = binaryData; + return true; + } + + if (data is not string encoded || encoded.Length > (((long)maximumBytes + 2) / 3 * 4) + 4) + { + return false; + } + + bytes = Convert.FromBase64String(encoded); + return bytes.Length <= maximumBytes; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Messages/SemanticSearch/SemanticSearchRequest.cs b/src/Microsoft.Health.Fhir.Core/Messages/SemanticSearch/SemanticSearchRequest.cs new file mode 100644 index 0000000000..40fc30bfd7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Messages/SemanticSearch/SemanticSearchRequest.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using EnsureThat; +using Medino; + +namespace Microsoft.Health.Fhir.Core.Messages.SemanticSearch +{ + /// + /// Requests semantic ranking of resources associated with a patient. + /// + public sealed class SemanticSearchRequest : IRequest + { + /// + /// Initializes a new instance of the class. + /// + public SemanticSearchRequest(string query, string patientId, int count, IReadOnlyCollection resourceTypes = null) + { + Query = EnsureArg.IsNotNullOrWhiteSpace(query, nameof(query)); + PatientId = EnsureArg.IsNotNullOrWhiteSpace(patientId, nameof(patientId)); + Count = EnsureArg.IsGt(count, 0, nameof(count)); + ResourceTypes = resourceTypes?.Distinct(StringComparer.Ordinal).ToArray() ?? Array.Empty(); + } + + /// Gets the natural-language query. + public string Query { get; } + + /// Gets the Patient resource ID used for compartment filtering. + public string PatientId { get; } + + /// Gets the maximum number of results. + public int Count { get; } + + /// Gets the selected FHIR resource types, or an empty collection when all supported types are requested. + public IReadOnlyCollection ResourceTypes { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Messages/SemanticSearch/SemanticSearchResponse.cs b/src/Microsoft.Health.Fhir.Core/Messages/SemanticSearch/SemanticSearchResponse.cs new file mode 100644 index 0000000000..cfef0aa817 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Messages/SemanticSearch/SemanticSearchResponse.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using EnsureThat; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Messages.SemanticSearch +{ + /// + /// Contains the FHIR search Bundle produced by semantic search. + /// + public sealed class SemanticSearchResponse + { + /// + /// Initializes a new instance of the class. + /// + public SemanticSearchResponse(ResourceElement bundle) + { + Bundle = EnsureArg.IsNotNull(bundle, nameof(bundle)); + } + + /// Gets the result Bundle. + public ResourceElement Bundle { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Microsoft.Health.Fhir.Core.csproj b/src/Microsoft.Health.Fhir.Core/Microsoft.Health.Fhir.Core.csproj index 0f46f641e9..479dfb6cf0 100644 --- a/src/Microsoft.Health.Fhir.Core/Microsoft.Health.Fhir.Core.csproj +++ b/src/Microsoft.Health.Fhir.Core/Microsoft.Health.Fhir.Core.csproj @@ -28,6 +28,7 @@ + @@ -64,6 +65,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Core/Models/SearchParameterInfo.cs b/src/Microsoft.Health.Fhir.Core/Models/SearchParameterInfo.cs index c9c262aaa7..9a11858990 100644 --- a/src/Microsoft.Health.Fhir.Core/Models/SearchParameterInfo.cs +++ b/src/Microsoft.Health.Fhir.Core/Models/SearchParameterInfo.cs @@ -24,6 +24,8 @@ public class SearchParameterInfo : IEquatable { public static readonly SearchParameterInfo ResourceTypeSearchParameter = new SearchParameterInfo(SearchParameterNames.ResourceType, SearchParameterNames.ResourceType, SearchParamType.Token, SearchParameterNames.ResourceTypeUri, null, "Resource.type().name", null); + public static readonly SearchParameterInfo ScoreSearchParameter = new SearchParameterInfo(SearchParameterNames.Score, SearchParameterNames.Score, SearchParamType.Special); + public SearchParameterInfo( string name, string code, @@ -33,7 +35,9 @@ public SearchParameterInfo( string expression = null, IReadOnlyList targetResourceTypes = null, IReadOnlyList baseResourceTypes = null, - string description = null) + string description = null, + VectorSearchParameterConfig vectorConfig = null, + string definitionStatus = null) : this(name, code) { Url = url; @@ -43,6 +47,8 @@ public SearchParameterInfo( TargetResourceTypes = targetResourceTypes; BaseResourceTypes = baseResourceTypes; Description = description; + VectorConfig = vectorConfig; + DefinitionStatus = definitionStatus; } public SearchParameterInfo(string name, string code) @@ -71,9 +77,11 @@ internal SearchParameterInfo(SearchParameterWrapper wrapper) Url = new Uri(wrapper.Url); Expression = wrapper.Expression; Description = wrapper.Description; + DefinitionStatus = wrapper.Status; Component = components; TargetResourceTypes = wrapper.Target; BaseResourceTypes = wrapper.Base; + VectorConfig = wrapper.VectorConfig; string GetComponentDefinition(ITypedElement component) { @@ -89,6 +97,11 @@ string GetComponentDefinition(ITypedElement component) public string Description { get; set; } + /// + /// Gets the publication status declared by the FHIR SearchParameter definition. + /// + public string DefinitionStatus { get; } + public string Expression { get; } public IReadOnlyList TargetResourceTypes { get; } = Array.Empty(); @@ -99,6 +112,11 @@ string GetComponentDefinition(ITypedElement component) public SearchParamType Type { get; set; } + /// + /// Gets vector-specific configuration when this definition is a vector SearchParameter. + /// + public VectorSearchParameterConfig VectorConfig { get; } + /// /// Returns true if this parameter is enabled for searches /// diff --git a/src/Microsoft.Health.Fhir.Core/Models/VectorSearchParameterConfig.cs b/src/Microsoft.Health.Fhir.Core/Models/VectorSearchParameterConfig.cs new file mode 100644 index 0000000000..e6e7210d25 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Models/VectorSearchParameterConfig.cs @@ -0,0 +1,88 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Models +{ + /// + /// Contains vector-specific metadata carried by a FHIR SearchParameter extension. + /// + public sealed class VectorSearchParameterConfig + { + /// + /// Gets the canonical URL of the vector SearchParameter configuration extension. + /// + public const string ExtensionUrl = "http://microsoft.com/fhir/StructureDefinition/vector-search-config"; + + /// + /// Gets the nested extension URL for the extraction policy. + /// + public const string ExtractionPolicyExtensionUrl = "extractionPolicy"; + + /// + /// Gets the nested extension URL for the source strategy. + /// + public const string SourceStrategyExtensionUrl = "sourceStrategy"; + + /// + /// Gets the nested extension URL for the maximum input token count. + /// + public const string MaxInputTokensExtensionUrl = "maxInputTokens"; + + /// + /// Gets the nested extension URL for the minimum normalized relevance score. + /// + public const string MinimumScoreExtensionUrl = "minimumScore"; + + /// + /// Gets the nested extension URL for the chunk size. + /// + public const string ChunkSizeTokensExtensionUrl = "chunkSizeTokens"; + + /// + /// Gets the nested extension URL for the chunk overlap. + /// + public const string ChunkOverlapTokensExtensionUrl = "chunkOverlapTokens"; + + /// + /// Gets the nested extension URL for the vector distance metric. + /// + public const string DistanceMetricExtensionUrl = "distanceMetric"; + + /// + /// Gets or sets the policy used to turn expression values into source passages. + /// + public VectorTextExtractionPolicy ExtractionPolicy { get; set; } = VectorTextExtractionPolicy.Concatenate; + + /// + /// Gets or sets the strategy used to resolve expression values to source text. + /// + public VectorTextSourceStrategy SourceStrategy { get; set; } = VectorTextSourceStrategy.DirectText; + + /// + /// Gets or sets the maximum number of source tokens accepted from this SearchParameter. + /// + public int MaxInputTokens { get; set; } = 8000; + + /// + /// Gets or sets the minimum normalized relevance score required for a chunk to match. + /// + public decimal MinimumScore { get; set; } + + /// + /// Gets or sets the optional chunk size. A null value uses the server default. + /// + public int? ChunkSizeTokens { get; set; } + + /// + /// Gets or sets the optional chunk overlap. A null value uses the server default. + /// + public int? ChunkOverlapTokens { get; set; } + + /// + /// Gets or sets the optional vector distance metric. A null value uses the server default. + /// + public string DistanceMetric { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Models/VectorTextExtractionPolicy.cs b/src/Microsoft.Health.Fhir.Core/Models/VectorTextExtractionPolicy.cs new file mode 100644 index 0000000000..955fcd0d12 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Models/VectorTextExtractionPolicy.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Models +{ + /// + /// Defines how values produced by a vector SearchParameter expression become source passages. + /// + public enum VectorTextExtractionPolicy + { + /// + /// Uses only the first value produced by the SearchParameter expression. + /// + FirstValue, + + /// + /// Concatenates all values produced by the SearchParameter expression. + /// + Concatenate, + + /// + /// Produces independent passages for each value produced by the SearchParameter expression. + /// + PerValueRow, + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Models/VectorTextSourceStrategy.cs b/src/Microsoft.Health.Fhir.Core/Models/VectorTextSourceStrategy.cs new file mode 100644 index 0000000000..77e95b02d2 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Models/VectorTextSourceStrategy.cs @@ -0,0 +1,23 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Models +{ + /// + /// Defines how values extracted by a vector SearchParameter are resolved to source text. + /// + public enum VectorTextSourceStrategy + { + /// + /// Uses each extracted value as source text. + /// + DirectText, + + /// + /// Treats extracted values as local Binary resource references and uses their text content. + /// + LocalBinaryReference, + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/ExpressionQueryBuilder.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/ExpressionQueryBuilder.cs index 9d3e737a9f..993039bd9f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/ExpressionQueryBuilder.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/ExpressionQueryBuilder.cs @@ -203,6 +203,11 @@ public object VisitSortParameter(SortExpression expression, Context context) throw new SearchOperationNotSupportedException(Microsoft.Health.Fhir.Core.Resources.SortNotSupported); } + public object VisitVectorSearch(VectorSearchExpression expression, Context context) + { + throw new SearchOperationNotSupportedException("Vector search is not supported for Cosmos DB."); + } + public object VisitMissingField(MissingFieldExpression expression, Context context) { _queryBuilder diff --git a/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Features/Definition/SemanticSearchParameterDefinitionTests.cs b/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Features/Definition/SemanticSearchParameterDefinitionTests.cs new file mode 100644 index 0000000000..a9090d3e1e --- /dev/null +++ b/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Features/Definition/SemanticSearchParameterDefinitionTests.cs @@ -0,0 +1,47 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Concurrent; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Shared.Core.Features.Search.Parameters; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Definition +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public sealed class SemanticSearchParameterDefinitionTests + { + [Fact] + public void GivenEmbeddedR4MicrosoftSearchParameters_WhenBuilt_ThenVectorDefinitionsAreNotSystemDefined() + { + var uriDictionary = new ConcurrentDictionary(); + var resourceTypeDictionary = new ConcurrentDictionary>>(); + var searchParameterComparer = new SearchParameterComparer(Substitute.For>>()); + var bundle = SearchParameterDefinitionBuilder.ReadEmbeddedSearchParameters("ms-search-parameters.json", ModelInfoProvider.Instance); + + SearchParameterDefinitionBuilder.Build( + bundle.Entries.Select(entry => entry.Resource).ToList(), + uriDictionary, + resourceTypeDictionary, + ModelInfoProvider.Instance, + searchParameterComparer, + NullLogger.Instance, + isSystemDefined: true); + + Assert.DoesNotContain(uriDictionary.Values, definition => definition.VectorConfig != null); + Assert.True(uriDictionary.TryGetValue("https://azurehealthcareapis.com/data-extensions/expiry-date", out SearchParameterInfo expiryDate)); + Assert.True(expiryDate.IsSystemDefined); + } + } +} diff --git a/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Features/Search/SemanticSearch/VectorTextSourceResolverTests.cs b/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Features/Search/SemanticSearch/VectorTextSourceResolverTests.cs new file mode 100644 index 0000000000..abe84a6178 --- /dev/null +++ b/src/Microsoft.Health.Fhir.R4.Core.UnitTests/Features/Search/SemanticSearch/VectorTextSourceResolverTests.cs @@ -0,0 +1,398 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Hl7.Fhir.Model; +using Hl7.Fhir.Serialization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Converters; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.ValueSets; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using UglyToad.PdfPig.Content; +using UglyToad.PdfPig.Core; +using UglyToad.PdfPig.Fonts.Standard14Fonts; +using UglyToad.PdfPig.Writer; +using Xunit; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public sealed class VectorTextSourceResolverTests + { + private static readonly Uri VectorCanonical = new Uri("https://example.org/fhir/SearchParameter/document-reference-binary-vector"); + + [Fact] + public async Task GivenDocumentReferenceAttachmentUrl_WhenExtractingVectorSearchParameter_ThenBinaryReferenceIsReturnedAsString() + { + // Arrange + SearchParameterInfo searchParameter = CreateSearchParameter("DocumentReference.content.attachment.url.toString()"); + ISupportedSearchParameterDefinitionManager definitionManager = Substitute.For(); + definitionManager.GetSearchParameters("DocumentReference").Returns(new[] { searchParameter }); + FhirTypedElementToSearchValueConverterManager converterManager = await SearchParameterFixtureData.GetFhirTypedElementToSearchValueConverterManagerAsync(); + var indexer = new TypedElementSearchIndexer( + definitionManager, + converterManager, + Substitute.For(), + ModelInfoProvider.Instance, + Substitute.For>()); + var documentReference = new DocumentReference + { + Content = new List + { + new DocumentReference.ContentComponent + { + Attachment = new Attachment { Url = "Binary/source" }, + }, + }, + }; + + // Act + IReadOnlyCollection searchIndices = indexer.Extract(documentReference.ToResourceElement()); + + // Assert + SearchIndexEntry searchIndex = Assert.Single(searchIndices); + Assert.Same(searchParameter, searchIndex.SearchParameter); + Assert.Equal("Binary/source", Assert.IsType(searchIndex.Value).String); + } + + [Fact] + public async Task GivenBinaryInWriteBatch_WhenResolvingReference_ThenBatchTextAndProvenanceAreReturned() + { + // Arrange + IVectorResourceReader resourceReader = Substitute.For(); + VectorTextSourceResolver resolver = CreateResolver(resourceReader); + ResourceWrapper owner = CreateResource("DocumentReference", "document", "1", "{}"); + ResourceWrapper binary = CreateBinary("source", "4", "same-batch text"); + + // Act + IReadOnlyList sources = await resolver.ResolveAsync( + owner, + CreateSearchParameter(), + new[] { "Binary/source" }, + new[] { owner, binary }, + CancellationToken.None); + + // Assert + VectorTextSource source = Assert.Single(sources); + Assert.Equal("same-batch text", source.Text); + Assert.Equal("Binary", source.ResourceType); + Assert.Equal("source", source.ResourceId); + Assert.Equal("4", source.ResourceVersion); + Assert.Equal("Binary.data", source.Path); + await resourceReader.DidNotReceiveWithAnyArgs().GetAsync(default, default); + } + + [Fact] + public async Task GivenPersistedBinary_WhenResolvingReference_ThenReaderTextIsReturned() + { + // Arrange + IVectorResourceReader resourceReader = Substitute.For(); + resourceReader.GetAsync( + Arg.Is(key => key.ResourceType == "Binary" && key.Id == "source"), + Arg.Any()) + .Returns(CreateBinary("source", "7", "persisted text")); + VectorTextSourceResolver resolver = CreateResolver(resourceReader); + ResourceWrapper owner = CreateResource("DocumentReference", "document", "1", "{}"); + + // Act + IReadOnlyList sources = await resolver.ResolveAsync( + owner, + CreateSearchParameter(), + new[] { "Binary/source" }, + new[] { owner }, + CancellationToken.None); + + // Assert + VectorTextSource source = Assert.Single(sources); + Assert.Equal("persisted text", source.Text); + Assert.Equal("7", source.ResourceVersion); + } + + [Fact] + public async Task GivenPdfBinary_WhenResolvingReference_ThenPageTextAndProvenanceAreReturned() + { + // Arrange + var resolver = new VectorTextSourceResolver( + Substitute.For(), + Deserializers.ResourceDeserializer, + new IBinaryContentExtractor[] + { + new PlainTextBinaryContentExtractor(), + new PdfBinaryContentExtractor(Options.Create(new VectorSearchConfiguration())), + }); + ResourceWrapper owner = CreateResource("DocumentReference", "document", "1", "{}"); + ResourceWrapper binary = CreateBinary("source", "3", "application/pdf", CreatePdf("first clinical page", "second clinical page")); + + // Act + IReadOnlyList sources = await resolver.ResolveAsync( + owner, + CreateSearchParameter(), + new[] { "Binary/source" }, + new[] { owner, binary }, + CancellationToken.None); + + // Assert + Assert.Collection( + sources, + source => + { + Assert.Contains("first clinical page", source.Text, StringComparison.Ordinal); + Assert.Equal("Binary.data#page=1", source.Path); + }, + source => + { + Assert.Contains("second clinical page", source.Text, StringComparison.Ordinal); + Assert.Equal("Binary.data#page=2", source.Path); + }); + Assert.All(sources, source => + { + Assert.Equal("Binary", source.ResourceType); + Assert.Equal("source", source.ResourceId); + Assert.Equal("3", source.ResourceVersion); + }); + } + + [Fact] + public async Task GivenSegmentedBinaryContent_WhenResolvingReference_ThenOrderedTextAndLocatorsAreReturned() + { + // Arrange + var extractor = new StubBinaryContentExtractor( + new BinaryContentSegment("first page", "page=1"), + new BinaryContentSegment("second page", "page=2")); + var resolver = new VectorTextSourceResolver( + Substitute.For(), + Deserializers.ResourceDeserializer, + new[] { extractor }); + ResourceWrapper owner = CreateResource("DocumentReference", "document", "1", "{}"); + ResourceWrapper binary = CreateResource( + "Binary", + "source", + "3", + "{\"resourceType\":\"Binary\",\"id\":\"source\",\"contentType\":\"application/pdf\",\"data\":\"cGRm\"}"); + + // Act + IReadOnlyList sources = await resolver.ResolveAsync( + owner, + CreateSearchParameter(), + new[] { "Binary/source" }, + new[] { owner, binary }, + CancellationToken.None); + + // Assert + Assert.Collection( + sources, + source => + { + Assert.Equal("first page", source.Text); + Assert.Equal("Binary.data#page=1", source.Path); + }, + source => + { + Assert.Equal("second page", source.Text); + Assert.Equal("Binary.data#page=2", source.Path); + }); + Assert.All(sources, source => + { + Assert.Equal("Binary", source.ResourceType); + Assert.Equal("source", source.ResourceId); + Assert.Equal("3", source.ResourceVersion); + }); + } + + [Fact] + public async Task GivenDocumentReferenceToBinary_WhenIndexing_ThenDecodedTextIsEmbeddedWithBinaryProvenance() + { + // Arrange + SearchParameterInfo searchParameter = CreateSearchParameter(); + ResourceWrapper owner = CreateResource( + "DocumentReference", + "document", + "2", + "{}", + new SearchIndexEntry(searchParameter, new StringSearchValue("Binary/source"))); + ResourceWrapper binary = CreateBinary("source", "5", "decoded clinical passage"); + IVectorSearchParameterResolver searchParameterResolver = Substitute.For(); + searchParameterResolver.GetIndexingSearchParameters("DocumentReference").Returns(new[] { searchParameter }); + searchParameterResolver.GetIndexingSearchParameters("Binary").Returns(Array.Empty()); + var embeddedTexts = new List(); + IEmbeddingClient embeddingClient = Substitute.For(); + embeddingClient.Dimensions.Returns(2); + embeddingClient.GenerateEmbeddingsAsync(Arg.Any>(), Arg.Any()) + .Returns(callInfo => + { + IReadOnlyList texts = callInfo.ArgAt>(0); + embeddedTexts.AddRange(texts); + return Task.FromResult>(texts.Select(_ => new[] { 0.25f, 0.75f }).ToList()); + }); + IEmbeddingModelRegistry modelRegistry = Substitute.For(); + modelRegistry.GetEmbeddingModelIdAsync(Arg.Any()).Returns((short)3); + var configuration = new VectorSearchConfiguration(); + configuration.Indexing.ChunkOverlapTokens = 0; + var indexer = new VectorSearchIndexer( + searchParameterResolver, + new TextChunker(), + embeddingClient, + modelRegistry, + CreateResolver(), + Options.Create(configuration), + NullLogger.Instance); + + // Act + await indexer.IndexAsync(new[] { owner, binary }, CancellationToken.None); + + // Assert + Assert.Equal(new[] { "decoded clinical passage" }, embeddedTexts); + VectorSearchChunk chunk = Assert.Single(Assert.Single(owner.VectorSearchIndices).Chunks); + Assert.Equal("Binary", chunk.SourceResourceType); + Assert.Equal("source", chunk.SourceResourceId); + Assert.Equal("5", chunk.SourceResourceVersion); + Assert.Equal("Binary.data", chunk.SourcePath); + } + + [Theory] + [InlineData("application/pdf", "ZmlsZSBjb250ZW50")] + [InlineData("text/plain; charset=iso-8859-1", "dGV4dA==")] + [InlineData("text/plain", "not-base64")] + public async Task GivenUnsupportedBinaryContent_WhenResolvingReference_ThenSourceIsSkipped(string contentType, string data) + { + // Arrange + VectorTextSourceResolver resolver = CreateResolver(); + ResourceWrapper owner = CreateResource("DocumentReference", "document", "1", "{}"); + ResourceWrapper binary = CreateResource( + "Binary", + "source", + "1", + $"{{\"resourceType\":\"Binary\",\"id\":\"source\",\"contentType\":\"{contentType}\",\"data\":\"{data}\"}}"); + + // Act + IReadOnlyList sources = await resolver.ResolveAsync( + owner, + CreateSearchParameter(), + new[] { "Binary/source" }, + new[] { owner, binary }, + CancellationToken.None); + + // Assert + Assert.Empty(sources); + } + + private static SearchParameterInfo CreateSearchParameter(string expression = "DocumentReference.content.attachment.url") + { + return new SearchParameterInfo( + name: "DocumentReferenceBinaryVector", + code: "binary-vector", + searchParamType: Microsoft.Health.Fhir.ValueSets.SearchParamType.Special, + url: VectorCanonical, + expression: expression, + baseResourceTypes: new[] { "DocumentReference" }, + vectorConfig: new VectorSearchParameterConfig + { + ExtractionPolicy = VectorTextExtractionPolicy.PerValueRow, + SourceStrategy = VectorTextSourceStrategy.LocalBinaryReference, + }, + definitionStatus: "active"); + } + + private static VectorTextSourceResolver CreateResolver(IVectorResourceReader resourceReader = null) + { + return new VectorTextSourceResolver( + resourceReader ?? Substitute.For(), + Deserializers.ResourceDeserializer, + new[] { new PlainTextBinaryContentExtractor() }); + } + + private static ResourceWrapper CreateBinary(string id, string version, string text) + { + return CreateBinary(id, version, "text/plain; charset=utf-8", Encoding.UTF8.GetBytes(text)); + } + + private static ResourceWrapper CreateBinary(string id, string version, string contentType, byte[] data) + { + var binary = new Binary { Id = id, ContentType = contentType, Data = data }; + + return CreateResource("Binary", id, version, new FhirJsonSerializer().SerializeToString(binary)); + } + + private static byte[] CreatePdf(params string[] pageTexts) + { + var builder = new PdfDocumentBuilder(); + PdfDocumentBuilder.AddedFont font = builder.AddStandard14Font(Standard14Font.Helvetica); + + foreach (string pageText in pageTexts) + { + PdfPageBuilder page = builder.AddPage(PageSize.A4); + page.AddText(pageText, 12, new PdfPoint(25, 700), font); + } + + return builder.Build(); + } + + private static ResourceWrapper CreateResource( + string resourceType, + string id, + string version, + string rawJson, + params SearchIndexEntry[] searchIndices) + { + return new ResourceWrapper( + resourceId: id, + versionId: version, + resourceTypeName: resourceType, + rawResource: new RawResource(rawJson, FhirResourceFormat.Json, isMetaSet: true), + request: new ResourceRequest("POST"), + lastModified: DateTimeOffset.UtcNow, + deleted: false, + searchIndices: searchIndices, + compartmentIndices: null, + lastModifiedClaims: Array.Empty>()); + } + + private sealed class StubBinaryContentExtractor : IBinaryContentExtractor + { + private readonly IReadOnlyList _segments; + + public StubBinaryContentExtractor(params BinaryContentSegment[] segments) + { + _segments = segments; + } + + public IReadOnlyCollection SupportedContentTypes { get; } = new[] { "application/pdf" }; + + public int GetMaximumContentLength(int maximumTextLength) + { + return maximumTextLength; + } + + public bool TryExtract( + byte[] content, + string contentType, + int maximumTextLength, + out IReadOnlyList segments) + { + segments = _segments; + return true; + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/SemanticSearchControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/SemanticSearchControllerTests.cs new file mode 100644 index 0000000000..11fff71053 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/SemanticSearchControllerTests.cs @@ -0,0 +1,169 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Linq; +using System.Net; +using System.Reflection; +using System.Threading; +using Hl7.Fhir.Model; +using Medino; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Routing; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Api.Controllers; +using Microsoft.Health.Fhir.Api.Features.ActionResults; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Messages.SemanticSearch; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Health.Fhir.Api.UnitTests.Controllers +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public class SemanticSearchControllerTests + { + private readonly IMediator _mediator = Substitute.For(); + private readonly SemanticSearchController _controller; + + public SemanticSearchControllerTests() + { + _controller = new SemanticSearchController( + _mediator, + Options.Create(new VectorSearchConfiguration())) + { + ControllerContext = new ControllerContext + { + HttpContext = new DefaultHttpContext(), + }, + }; + } + + [Fact] + public async Task GivenPatientSemanticSearch_WhenValid_ThenPatientIdComesFromRoute() + { + _mediator.SendAsync(Arg.Any(), Arg.Any()) + .Returns(new SemanticSearchResponse(new Bundle { Type = Bundle.BundleType.Searchset }.ToResourceElement())); + var parameters = new Parameters + { + Parameter = + { + new Parameters.ParameterComponent { Name = "query", Value = new FhirString("breathing difficulty") }, + new Parameters.ParameterComponent { Name = "count", Value = new Integer(3) }, + new Parameters.ParameterComponent { Name = "type", Value = new Code("Observation") }, + new Parameters.ParameterComponent { Name = "type", Value = new Code("DiagnosticReport") }, + new Parameters.ParameterComponent { Name = "patient", Value = new ResourceReference("Patient/ignored") }, + }, + }; + + var result = await _controller.Search("123", parameters) as FhirResult; + + Assert.NotNull(result); + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + await _mediator.Received(1).SendAsync( + Arg.Is(request => + request.Query == "breathing difficulty" && + request.PatientId == "123" && + request.Count == 3 && + request.ResourceTypes.SequenceEqual(new[] { "Observation", "DiagnosticReport" })), + Arg.Any()); + } + + [Fact] + public async Task GivenPatientSemanticSearchWithoutCount_WhenValid_ThenDefaultCountIsUsed() + { + _mediator.SendAsync(Arg.Any(), Arg.Any()) + .Returns(new SemanticSearchResponse(new Bundle { Type = Bundle.BundleType.Searchset }.ToResourceElement())); + var parameters = new Parameters + { + Parameter = + { + new Parameters.ParameterComponent { Name = "query", Value = new FhirString("breathing difficulty") }, + }, + }; + + await _controller.Search("123", parameters); + + await _mediator.Received(1).SendAsync( + Arg.Is(request => request.Count == 10), + Arg.Any()); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task GivenPatientSemanticSearchWithoutQuery_WhenHandled_ThenRequestIsRejected(string query) + { + var parameters = new Parameters + { + Parameter = + { + new Parameters.ParameterComponent { Name = "query", Value = new FhirString(query) }, + }, + }; + + await Assert.ThrowsAsync(() => _controller.Search("123", parameters)); + await _mediator.DidNotReceive().SendAsync(Arg.Any(), Arg.Any()); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(51)] + public async Task GivenPatientSemanticSearchWithInvalidCount_WhenHandled_ThenRequestIsRejected(int count) + { + var parameters = new Parameters + { + Parameter = + { + new Parameters.ParameterComponent { Name = "query", Value = new FhirString("breathing difficulty") }, + new Parameters.ParameterComponent { Name = "count", Value = new Integer(count) }, + }, + }; + + await Assert.ThrowsAsync(() => _controller.Search("123", parameters)); + await _mediator.DidNotReceive().SendAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task GivenPatientSemanticSearchWithResourceType_WhenHandled_ThenTypeIsForwardedForMetadataValidation() + { + _mediator.SendAsync(Arg.Any(), Arg.Any()) + .Returns(new SemanticSearchResponse(new Bundle { Type = Bundle.BundleType.Searchset }.ToResourceElement())); + var parameters = new Parameters + { + Parameter = + { + new Parameters.ParameterComponent { Name = "query", Value = new FhirString("breathing difficulty") }, + new Parameters.ParameterComponent { Name = "type", Value = new Code("Condition") }, + }, + }; + + await _controller.Search("123", parameters); + + await _mediator.Received(1).SendAsync( + Arg.Is(request => request.ResourceTypes.SequenceEqual(new[] { "Condition" })), + Arg.Any()); + } + + [Fact] + public void GivenSemanticSearchController_WhenInspectingRoute_ThenPatientInstanceOperationIsExposed() + { + RouteAttribute route = typeof(SemanticSearchController) + .GetMethod(nameof(SemanticSearchController.Search), BindingFlags.Instance | BindingFlags.Public) + .GetCustomAttributes() + .Single(); + + Assert.Equal("Patient/{idParameter}/$semantic-search", route.Template); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Formatters/FhirJsonOutputFormatterTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Formatters/FhirJsonOutputFormatterTests.cs index b1b9889ff8..2e4771fbfe 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Formatters/FhirJsonOutputFormatterTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Formatters/FhirJsonOutputFormatterTests.cs @@ -21,6 +21,7 @@ using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Shared.Core.Features.Search; using Microsoft.Health.Fhir.Tests.Common; @@ -103,6 +104,49 @@ public async Task GivenContext_WhenWritingResponseBody_ThenResourceShouldBeWritt await Run(false, raw, query); } + [Fact] + public async Task GivenRawSearchBundleWithSemanticMetadata_WhenWritingResponse_ThenScoreAndEvidenceShouldBeWritten() + { + using var writer = new StringWriter(new StringBuilder()); + using var body = new MemoryStream(); + var httpContext = new DefaultHttpContext(); + httpContext.Response.Body = body; + var bundle = (Hl7.Fhir.Model.Bundle)CreateObject(bundle: true, raw: true); + var evidence = new Extension { Url = SemanticSearchEvidence.ExtensionUrl }; + evidence.Extension.Add(new Extension(SemanticSearchEvidence.TextExtensionUrl, new FhirString("Matched passage"))); + evidence.Extension.Add(new Extension(SemanticSearchEvidence.RankExtensionUrl, new PositiveInt(1))); + bundle.Entry.Single().Search = new SearchComponent + { + Mode = SearchEntryMode.Match, + Score = 0.91m, + }; + bundle.Entry.Single().Search.Extension.Add(evidence); + var writeContext = new OutputFormatterWriteContext( + httpContext, + (_, _) => writer, + typeof(Hl7.Fhir.Model.Bundle), + bundle); + var formatter = new FhirJsonOutputFormatter( + new FhirJsonSerializer(), + Deserializers.ResourceDeserializer, + ArrayPool.Shared, + new BundleSerializer(), + ModelInfoProvider.Instance); + + await formatter.WriteResponseBodyAsync(writeContext, Encoding.UTF8); + + Hl7.Fhir.Model.Bundle serializedBundle = Parser.Parse(writer.ToString()); + SearchComponent search = Assert.Single(serializedBundle.Entry).Search; + Assert.Equal(0.91m, search.Score); + Extension serializedEvidence = Assert.Single(search.Extension, extension => extension.Url == SemanticSearchEvidence.ExtensionUrl); + Assert.Equal( + "Matched passage", + ((FhirString)Assert.Single(serializedEvidence.Extension, extension => extension.Url == SemanticSearchEvidence.TextExtensionUrl).Value).Value); + Assert.Equal( + 1, + ((PositiveInt)Assert.Single(serializedEvidence.Extension, extension => extension.Url == SemanticSearchEvidence.RankExtensionUrl).Value).Value); + } + private static async Task Run( bool bundle, bool raw, diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerTests.cs index 861dc62cde..c241070099 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerTests.cs @@ -37,6 +37,7 @@ using Microsoft.Health.Fhir.Core.Features.Resources.Bundle; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Parameters; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Features.Validation; using Microsoft.Health.Fhir.Core.Logging.Metrics; @@ -64,6 +65,7 @@ public class BundleHandlerTests private readonly IMediator _mediator; private readonly IBundleMetricHandler _bundleMetricHandler; private readonly ITransactionHandler _transactionHandler; + private readonly IVectorSearchParameterResolver _vectorSearchParameterResolver; private DefaultFhirRequestContext _fhirRequestContext; private readonly IProvideProfilesForValidation _profilesResolver; @@ -128,6 +130,7 @@ public BundleHandlerTests() _mediator = Substitute.For(); _bundleMetricHandler = Substitute.For(); + _vectorSearchParameterResolver = Substitute.For(); _bundleHandler = new BundleHandler( httpContextAccessor, @@ -149,7 +152,62 @@ public BundleHandlerTests() _mediator, _router, _bundleMetricHandler, - NullLogger.Instance); + NullLogger.Instance, + _vectorSearchParameterResolver); + } + + [Fact] + public async Task GivenSequentialTransactionWithLocalReferenceVectorOwner_WhenHandled_ThenParallelProcessingIsUsed() + { + // Arrange + _bundleConfiguration.TransactionDefaultProcessingLogic = BundleProcessingLogic.Sequential; + _bundleConfiguration.SupportsBundleOrchestrator = true; + var searchParameter = new SearchParameterInfo( + name: "DocumentReferenceSemanticText", + code: "semantic-text", + searchParamType: Microsoft.Health.Fhir.ValueSets.SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/document-reference-semantic-text"), + expression: "DocumentReference.content.attachment.url.toString()", + baseResourceTypes: new[] { "DocumentReference" }, + vectorConfig: new VectorSearchParameterConfig + { + SourceStrategy = VectorTextSourceStrategy.LocalBinaryReference, + }, + definitionStatus: "active"); + _vectorSearchParameterResolver.GetIndexingSearchParameters("DocumentReference").Returns(new[] { searchParameter }); + var bundle = new Hl7.Fhir.Model.Bundle + { + Type = BundleType.Transaction, + Entry = + { + new EntryComponent + { + Resource = new DocumentReference(), + Request = new RequestComponent { Method = HTTPVerb.POST, Url = "DocumentReference" }, + }, + new EntryComponent + { + Resource = new Binary(), + Request = new RequestComponent { Method = HTTPVerb.POST, Url = "Binary" }, + }, + }, + }; + _router.When(router => router.RouteAsync(Arg.Any())) + .Do(callInfo => + { + callInfo.Arg().Handler = context => + { + context.Response.StatusCode = StatusCodes.Status200OK; + return Task.CompletedTask; + }; + }); + + // Act + BundleResponse response = await _bundleHandler.HandleAsync(new BundleRequest(bundle.ToResourceElement()), CancellationToken.None); + + // Assert + Assert.Equal(BundleProcessingLogic.Parallel, response.Info.ProcessingLogic); + _transactionHandler.DidNotReceive().BeginTransaction(); } [Fact] diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems index 75aeff4664..f4a2852df8 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems @@ -26,6 +26,7 @@ + @@ -101,6 +102,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Registration/FhirServerServiceCollectionExtensionsTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Registration/FhirServerServiceCollectionExtensionsTests.cs new file mode 100644 index 0000000000..564b64614c --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Registration/FhirServerServiceCollectionExtensionsTests.cs @@ -0,0 +1,97 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Api.UnitTests.Registration +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public sealed class FhirServerServiceCollectionExtensionsTests + { + [Fact] + public void GivenCompleteVectorSearchSettings_WhenFhirServerIsAdded_ThenSettingsAreBoundAndRegistered() + { + // Arrange + IConfiguration configuration = BuildConfiguration(new Dictionary + { + ["FhirServer:CoreFeatures:VectorSearch:Enabled"] = "true", + ["FhirServer:CoreFeatures:VectorSearch:Embedding:Endpoint"] = "https://embedding.example.com", + ["FhirServer:CoreFeatures:VectorSearch:Embedding:DeploymentName"] = "embedding-deployment", + ["FhirServer:CoreFeatures:VectorSearch:Embedding:ModelName"] = "embedding-model", + ["FhirServer:CoreFeatures:VectorSearch:Embedding:ModelVersion"] = "1", + ["FhirServer:CoreFeatures:VectorSearch:Embedding:Dimensions"] = "1536", + ["FhirServer:CoreFeatures:VectorSearch:Indexing:Mode"] = "Synchronous", + ["FhirServer:CoreFeatures:VectorSearch:Indexing:ChunkSizeTokens"] = "600", + ["FhirServer:CoreFeatures:VectorSearch:Indexing:ChunkOverlapTokens"] = "50", + ["FhirServer:CoreFeatures:VectorSearch:Indexing:Pdf:MaximumFileSizeBytes"] = "5242880", + ["FhirServer:CoreFeatures:VectorSearch:Indexing:Pdf:MaximumPageCount"] = "100", + ["FhirServer:CoreFeatures:VectorSearch:Indexing:Pdf:MaximumExtractedCharacters"] = "250000", + ["FhirServer:CoreFeatures:VectorSearch:Indexing:Pdf:ExtractionTimeout"] = "00:00:15", + ["FhirServer:CoreFeatures:VectorSearch:Query:DefaultCount"] = "5", + ["FhirServer:CoreFeatures:VectorSearch:Query:MaxCount"] = "25", + ["FhirServer:CoreFeatures:VectorSearch:Query:CandidateCount"] = "125", + ["FhirServer:CoreFeatures:VectorSearch:Query:DistanceMetric"] = "cosine", + }); + var services = new ServiceCollection(); + + // Act + services.AddFhirServer(configuration); + using ServiceProvider serviceProvider = services.BuildServiceProvider(); + VectorSearchConfiguration vectorSearch = serviceProvider.GetRequiredService>().Value; + + // Assert + Assert.True(vectorSearch.Enabled); + Assert.Equal(new Uri("https://embedding.example.com"), vectorSearch.Embedding.Endpoint); + Assert.Equal("embedding-deployment", vectorSearch.Embedding.DeploymentName); + Assert.Equal("embedding-model", vectorSearch.Embedding.ModelName); + Assert.Equal("1", vectorSearch.Embedding.ModelVersion); + Assert.Equal(1536, vectorSearch.Embedding.Dimensions); + Assert.Equal(VectorSearchIndexingMode.Synchronous, vectorSearch.Indexing.Mode); + Assert.Equal(600, vectorSearch.Indexing.ChunkSizeTokens); + Assert.Equal(50, vectorSearch.Indexing.ChunkOverlapTokens); + Assert.Equal(5 * 1024 * 1024, vectorSearch.Indexing.Pdf.MaximumFileSizeBytes); + Assert.Equal(100, vectorSearch.Indexing.Pdf.MaximumPageCount); + Assert.Equal(250_000, vectorSearch.Indexing.Pdf.MaximumExtractedCharacters); + Assert.Equal(TimeSpan.FromSeconds(15), vectorSearch.Indexing.Pdf.ExtractionTimeout); + Assert.Equal(5, vectorSearch.Query.DefaultCount); + Assert.Equal(25, vectorSearch.Query.MaxCount); + Assert.Equal(125, vectorSearch.Query.CandidateCount); + Assert.Equal("cosine", vectorSearch.Query.DistanceMetric); + } + + [Fact] + public void GivenIncompleteEnabledVectorSearchSettings_WhenFhirServerIsAdded_ThenStartupValidationFails() + { + // Arrange + IConfiguration configuration = BuildConfiguration(new Dictionary + { + ["FhirServer:CoreFeatures:VectorSearch:Enabled"] = "true", + }); + var services = new ServiceCollection(); + + // Act + Action addFhirServer = () => services.AddFhirServer(configuration); + + // Assert + Assert.Throws(addFhirServer); + } + + private static IConfiguration BuildConfiguration(IDictionary settings) + { + return new ConfigurationBuilder() + .AddInMemoryCollection(settings) + .Build(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/SemanticSearchController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/SemanticSearchController.cs new file mode 100644 index 0000000000..c9bac2d2a9 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/SemanticSearchController.cs @@ -0,0 +1,88 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using EnsureThat; +using Hl7.Fhir.Model; +using Medino; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; +using Microsoft.Health.Api.Features.Audit; +using Microsoft.Health.Fhir.Api.Features.ActionResults; +using Microsoft.Health.Fhir.Api.Features.Filters; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Messages.SemanticSearch; +using Microsoft.Health.Fhir.ValueSets; + +namespace Microsoft.Health.Fhir.Api.Controllers +{ + /// + /// Provides semantic search over resources in a patient compartment. + /// + [ServiceFilter(typeof(OperationOutcomeExceptionFilterAttribute))] + [ServiceFilter(typeof(ValidateFormatParametersAttribute))] + [ValidateModelState] + public sealed class SemanticSearchController : Controller + { + private readonly IMediator _mediator; + private readonly VectorSearchQueryConfiguration _queryConfiguration; + + /// + /// Initializes a new instance of the class. + /// + /// The mediator used to dispatch the semantic-search request. + /// The vector-search configuration. + public SemanticSearchController(IMediator mediator, IOptions configuration) + { + _mediator = EnsureArg.IsNotNull(mediator, nameof(mediator)); + _queryConfiguration = EnsureArg.IsNotNull(configuration, nameof(configuration)).Value.Query; + } + + /// + /// Semantically searches resources associated with a patient. + /// + /// The patient resource ID. + /// The semantic-search operation parameters. + [HttpPost] + [Route(KnownRoutes.SemanticSearchPatientById)] + [AuditEventType(AuditEventSubType.SearchSystem)] + public async Task Search(string idParameter, [FromBody] Parameters parameters) + { + FhirString query = parameters?.Parameter?.FirstOrDefault(parameter => parameter.Name == "query")?.Value as FhirString; + Integer count = parameters?.Parameter?.FirstOrDefault(parameter => parameter.Name == "count")?.Value as Integer; + IReadOnlyList resourceTypes = parameters?.Parameter? + .Where(parameter => parameter.Name == "type") + .Select(parameter => (parameter.Value as Code)?.Value) + .Where(resourceType => !string.IsNullOrWhiteSpace(resourceType)) + .Distinct(StringComparer.Ordinal) + .ToList() ?? new List(); + + if (string.IsNullOrWhiteSpace(query?.Value)) + { + throw new RequestNotValidException("Semantic search requires a query parameter."); + } + + if (count?.Value <= 0) + { + throw new RequestNotValidException("Semantic search count must be greater than zero."); + } + + if (count?.Value > _queryConfiguration.MaxCount) + { + throw new RequestNotValidException($"Semantic search count must not exceed {_queryConfiguration.MaxCount}."); + } + + SemanticSearchResponse response = await _mediator.SendAsync( + new SemanticSearchRequest(query.Value, idParameter, count?.Value ?? _queryConfiguration.DefaultCount, resourceTypes), + HttpContext.RequestAborted); + return FhirResult.Create(response.Bundle); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Features/Formatters/FhirJsonOutputFormatter.cs b/src/Microsoft.Health.Fhir.Shared.Api/Features/Formatters/FhirJsonOutputFormatter.cs index c0edf3eb93..d49d6a3173 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Features/Formatters/FhirJsonOutputFormatter.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Features/Formatters/FhirJsonOutputFormatter.cs @@ -86,12 +86,16 @@ public override async Task WriteResponseBodyAsync(OutputFormatterWriteContext co { var bundle = context.Object as Hl7.Fhir.Model.Bundle; resource = bundle; + bool hasExtendedSearchMetadata = bundle.Entry + .OfType() + .Any(entry => entry.Search?.Score.HasValue == true || entry.Search?.Extension?.Any() == true); if (hasElements || summarySearchParameter != Hl7.Fhir.Rest.SummaryType.False || - !bundle.Entry.All(x => x is RawBundleEntryComponent)) + !bundle.Entry.All(x => x is RawBundleEntryComponent) || + hasExtendedSearchMetadata) { - // _elements is not supported for a raw resource, revert to using FhirJsonSerializer + // The raw serializer does not support filtered resources or extended search metadata. foreach (var rawBundleEntryComponent in bundle.Entry) { if (rawBundleEntryComponent is RawBundleEntryComponent) diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs b/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs index 6555c41b35..85215239e1 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs @@ -49,6 +49,7 @@ using Microsoft.Health.Fhir.Core.Features.Resources.Bundle; using Microsoft.Health.Fhir.Core.Features.Routing; using Microsoft.Health.Fhir.Core.Features.Search.Parameters; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Features.Security; using Microsoft.Health.Fhir.Core.Features.Validation; using Microsoft.Health.Fhir.Core.Logging.Metrics; @@ -91,6 +92,7 @@ public partial class BundleHandler : IRequestHandler _logger; @@ -145,7 +147,8 @@ public BundleHandler( IMediator mediator, IRouter router, IBundleMetricHandler metricHandler, - ILogger logger) + ILogger logger, + IVectorSearchParameterResolver vectorSearchParameterResolver = null) : this() { EnsureArg.IsNotNull(httpContextAccessor, nameof(httpContextAccessor)); @@ -167,6 +170,7 @@ public BundleHandler( _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _modelInfoProvider = EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider)); _searchParameterOperations = EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); + _vectorSearchParameterResolver = vectorSearchParameterResolver; _metricHandler = EnsureArg.IsNotNull(metricHandler, nameof(metricHandler)); // Not all versions support the same enum values, so do the dictionary creation in the version specific partial. @@ -261,6 +265,11 @@ public async Task HandleAsync(BundleRequest request, Cancellatio _logger.LogInformation("Sequential transaction bundle contains a search parameter, execution is forced to be parallel."); bundleProcessingLogic = BundleProcessingLogic.Parallel; } + else if (_vectorSearchParameterResolver != null && ContainsLocalReferenceVectorOwner(bundleResource, _vectorSearchParameterResolver)) + { + _logger.LogInformation("Sequential transaction bundle contains a local-reference vector owner, execution is forced to be parallel."); + bundleProcessingLogic = BundleProcessingLogic.Parallel; + } } var responseBundle = new Hl7.Fhir.Model.Bundle { Type = BundleType.TransactionResponse }; @@ -288,6 +297,18 @@ public async Task HandleAsync(BundleRequest request, Cancellatio } } + private static bool ContainsLocalReferenceVectorOwner( + Hl7.Fhir.Model.Bundle bundle, + IVectorSearchParameterResolver vectorSearchParameterResolver) + { + return bundle.Entry + .Select(entry => entry.Resource?.TypeName) + .Where(resourceType => resourceType != null) + .Distinct(StringComparer.Ordinal) + .SelectMany(vectorSearchParameterResolver.GetIndexingSearchParameters) + .Any(searchParameter => searchParameter.VectorConfig.SourceStrategy == VectorTextSourceStrategy.LocalBinaryReference); + } + private async Task CheckSearchParamInputConflictsAndUpdateCache(Hl7.Fhir.Model.Bundle bundle, CancellationToken cancellationToken) { var codes = new HashSet<(string Type, string Code)>(); diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems b/src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems index 5ec9859b13..9967f6c091 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems @@ -22,6 +22,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Modules/SearchModule.cs b/src/Microsoft.Health.Fhir.Shared.Api/Modules/SearchModule.cs index 387d502ed0..e4bed6ff14 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Modules/SearchModule.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Modules/SearchModule.cs @@ -26,6 +26,7 @@ using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Messages.Create; using Microsoft.Health.Fhir.Core.Messages.Delete; using Microsoft.Health.Fhir.Core.Messages.Search; @@ -165,6 +166,7 @@ public void Load(IServiceCollection services) services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddTransient, CreateOrUpdateSearchParameterBehavior>(); services.AddTransient, CreateOrUpdateSearchParameterBehavior>(); services.AddTransient, DeleteSearchParameterBehavior>(); diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs b/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs index b248dcf990..9f67e05782 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs @@ -83,10 +83,13 @@ public static IFhirServerBuilder AddFhirServer( configurationRoot?.GetSection(FhirServerConfigurationSectionName).Bind(fhirServerConfiguration); configureAction?.Invoke(fhirServerConfiguration); + fhirServerConfiguration.CoreFeatures.VectorSearch.Validate(); + services.AddSingleton(Options.Options.Create(fhirServerConfiguration)); services.AddSingleton(Options.Options.Create(fhirServerConfiguration.Security)); services.AddSingleton(Options.Options.Create(fhirServerConfiguration.Features)); services.AddSingleton(Options.Options.Create(fhirServerConfiguration.CoreFeatures)); + services.AddSingleton(Options.Options.Create(fhirServerConfiguration.CoreFeatures.VectorSearch)); services.AddSingleton(Options.Options.Create(fhirServerConfiguration.Cors)); services.AddSingleton(Options.Options.Create(fhirServerConfiguration.Operations)); services.AddSingleton(Options.Options.Create(fhirServerConfiguration.Operations.Export)); diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Definition/SearchParameterDefinitionBuilderTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Definition/SearchParameterDefinitionBuilderTests.cs index 50f1cc04ad..aaf5865bd9 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Definition/SearchParameterDefinitionBuilderTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Definition/SearchParameterDefinitionBuilderTests.cs @@ -99,6 +99,81 @@ public void GivenAValidSearchParameterDefinitionFile_WhenBuilt_ThenUriDictionary _uriDictionary.Values.Select(value => value.Url.ToString()).OrderBy(s => s, StringComparer.OrdinalIgnoreCase)); } + [Fact] + public void GivenVectorSearchParameterExtension_WhenWrapped_ThenVectorConfigurationIsParsed() + { + // Arrange + const string searchParameterJson = "{\"resourceType\":\"SearchParameter\",\"url\":\"https://example.org/fhir/SearchParameter/observation-note-vector\",\"name\":\"ObservationNoteVector\",\"status\":\"active\",\"code\":\"note-vector\",\"base\":[\"Observation\"],\"type\":\"special\",\"expression\":\"Observation.note.text\",\"extension\":[{\"url\":\"http://microsoft.com/fhir/StructureDefinition/vector-search-config\",\"extension\":[{\"url\":\"sourceStrategy\",\"valueCode\":\"localBinaryReference\"},{\"url\":\"extractionPolicy\",\"valueCode\":\"perValueRow\"},{\"url\":\"maxInputTokens\",\"valueInteger\":1200},{\"url\":\"minimumScore\",\"valueDecimal\":0.65},{\"url\":\"chunkSizeTokens\",\"valueInteger\":400},{\"url\":\"chunkOverlapTokens\",\"valueInteger\":40},{\"url\":\"distanceMetric\",\"valueCode\":\"cosine\"}]}]}"; + SearchParameter searchParameter = _jsonParser.Parse(searchParameterJson); + + // Act + var searchParameterInfo = new SearchParameterInfo(new SearchParameterWrapper(searchParameter.ToTypedElement())); + + // Assert + Assert.Equal("active", searchParameterInfo.DefinitionStatus); + Assert.NotNull(searchParameterInfo.VectorConfig); + Assert.Equal(VectorTextSourceStrategy.LocalBinaryReference, searchParameterInfo.VectorConfig.SourceStrategy); + Assert.Equal(VectorTextExtractionPolicy.PerValueRow, searchParameterInfo.VectorConfig.ExtractionPolicy); + Assert.Equal(1200, searchParameterInfo.VectorConfig.MaxInputTokens); + Assert.Equal(0.65m, searchParameterInfo.VectorConfig.MinimumScore); + Assert.Equal(400, searchParameterInfo.VectorConfig.ChunkSizeTokens); + Assert.Equal(40, searchParameterInfo.VectorConfig.ChunkOverlapTokens); + Assert.Equal("cosine", searchParameterInfo.VectorConfig.DistanceMetric); + } + + [Theory] + [InlineData("chunkSizeTokens", "valueInteger", "0")] + [InlineData("chunkOverlapTokens", "valueInteger", "-1")] + [InlineData("distanceMetric", "valueCode", "euclidean")] + public void GivenInvalidVectorIndexSetting_WhenWrapped_ThenDefinitionIsRejected(string setting, string valueType, string value) + { + string searchParameterJson = $"{{\"resourceType\":\"SearchParameter\",\"url\":\"https://example.org/fhir/SearchParameter/observation-note-vector\",\"name\":\"ObservationNoteVector\",\"status\":\"active\",\"code\":\"note-vector\",\"base\":[\"Observation\"],\"type\":\"special\",\"expression\":\"Observation.note.text\",\"extension\":[{{\"url\":\"http://microsoft.com/fhir/StructureDefinition/vector-search-config\",\"extension\":[{{\"url\":\"{setting}\",\"{valueType}\":{(valueType == "valueCode" ? $"\"{value}\"" : value)}}}]}}]}}"; + SearchParameter searchParameter = _jsonParser.Parse(searchParameterJson); + Action wrap = () => new SearchParameterInfo(new SearchParameterWrapper(searchParameter.ToTypedElement())); + + Assert.Throws(wrap); + } + + [Fact] + public void GivenChunkOverlapNotSmallerThanChunkSize_WhenWrapped_ThenDefinitionIsRejected() + { + const string searchParameterJson = "{\"resourceType\":\"SearchParameter\",\"url\":\"https://example.org/fhir/SearchParameter/observation-note-vector\",\"name\":\"ObservationNoteVector\",\"status\":\"active\",\"code\":\"note-vector\",\"base\":[\"Observation\"],\"type\":\"special\",\"expression\":\"Observation.note.text\",\"extension\":[{\"url\":\"http://microsoft.com/fhir/StructureDefinition/vector-search-config\",\"extension\":[{\"url\":\"chunkSizeTokens\",\"valueInteger\":100},{\"url\":\"chunkOverlapTokens\",\"valueInteger\":100}]}]}"; + SearchParameter searchParameter = _jsonParser.Parse(searchParameterJson); + Action wrap = () => new SearchParameterInfo(new SearchParameterWrapper(searchParameter.ToTypedElement())); + + Assert.Throws(wrap); + } + + [Theory] + [InlineData("-0.01")] + [InlineData("1.01")] + public void GivenInvalidVectorMinimumScore_WhenWrapped_ThenDefinitionIsRejected(string minimumScore) + { + // Arrange + string searchParameterJson = $"{{\"resourceType\":\"SearchParameter\",\"url\":\"https://example.org/fhir/SearchParameter/observation-note-vector\",\"name\":\"ObservationNoteVector\",\"status\":\"active\",\"code\":\"note-vector\",\"base\":[\"Observation\"],\"type\":\"special\",\"expression\":\"Observation.note.text\",\"extension\":[{{\"url\":\"http://microsoft.com/fhir/StructureDefinition/vector-search-config\",\"extension\":[{{\"url\":\"minimumScore\",\"valueDecimal\":{minimumScore}}}]}}]}}"; + SearchParameter searchParameter = _jsonParser.Parse(searchParameterJson); + + // Act + Action wrap = () => new SearchParameterInfo(new SearchParameterWrapper(searchParameter.ToTypedElement())); + + // Assert + Assert.Throws(wrap); + } + + [Fact] + public void GivenUnsupportedVectorExtractionPolicy_WhenWrapped_ThenDefinitionIsRejected() + { + // Arrange + const string searchParameterJson = "{\"resourceType\":\"SearchParameter\",\"url\":\"https://example.org/fhir/SearchParameter/observation-note-vector\",\"name\":\"ObservationNoteVector\",\"status\":\"active\",\"code\":\"note-vector\",\"base\":[\"Observation\"],\"type\":\"special\",\"expression\":\"Observation.note.text\",\"extension\":[{\"url\":\"http://microsoft.com/fhir/StructureDefinition/vector-search-config\",\"extension\":[{\"url\":\"extractionPolicy\",\"valueCode\":\"unsupported\"}]}]}"; + SearchParameter searchParameter = _jsonParser.Parse(searchParameterJson); + + // Act + Action wrap = () => new SearchParameterInfo(new SearchParameterWrapper(searchParameter.ToTypedElement())); + + // Assert + Assert.Throws(wrap); + } + [Fact] public void GivenAValidSearchParameterDefinitionFile_WhenBuilt_ThenAllResourceTypesShouldBeIncluded() { diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/BundleFactoryTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/BundleFactoryTests.cs index 37cd1880d8..62f9ee0840 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/BundleFactoryTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/BundleFactoryTests.cs @@ -17,6 +17,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Routing; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Shared.Core.Features.Search; using Microsoft.Health.Fhir.Tests.Common; @@ -141,6 +142,44 @@ async Task ValidateEntry(Observation expected, Bundle.EntryComponent actualEntry } } + [Fact] + public void GivenASemanticSearchResult_WhenCreateSearchBundle_ThenScoreAndEvidenceAreReturned() + { + _urlResolver.ResolveResourceWrapperUrl(Arg.Any()).Returns(new Uri("http://resource/123")); + _urlResolver.ResolveRouteUrl(_unsupportedSearchParameters).Returns(_selfUrl); + + ResourceElement observation = Samples.GetDefaultObservation().UpdateId("123"); + var evidence = new SemanticSearchEvidence( + "Patient reports shortness of breath while climbing stairs.", + chunkOrdinal: 2, + score: 0.91m, + new Uri("https://example.org/fhir/SearchParameter/semantic-text"), + "Observation/123/_history/4", + "Observation.note.text", + rank: 2, + witnessReference: "DocumentReference/document/_history/3"); + var searchResult = new SearchResult( + new[] { new SearchResultEntry(CreateResourceWrapper(observation, HttpMethod.Post), score: 0.91m, evidence: evidence) }, + continuationToken: null, + sortOrder: null, + unsupportedSearchParameters: _unsupportedSearchParameters); + + Bundle bundle = _bundleFactory.CreateSearchBundle(searchResult).ToPoco(); + Bundle.SearchComponent search = Assert.Single(bundle.Entry).Search; + + Assert.Equal(0.91m, search.Score); + Extension evidenceExtension = Assert.Single(search.Extension, extension => extension.Url == SemanticSearchEvidence.ExtensionUrl); + Assert.Equal(evidence.Text, ((FhirString)evidenceExtension.Extension.Single(extension => extension.Url == SemanticSearchEvidence.TextExtensionUrl).Value).Value); + Assert.Equal(evidence.ChunkOrdinal, ((Integer)evidenceExtension.Extension.Single(extension => extension.Url == SemanticSearchEvidence.ChunkOrdinalExtensionUrl).Value).Value); + Assert.Equal(evidence.Rank, ((PositiveInt)evidenceExtension.Extension.Single(extension => extension.Url == SemanticSearchEvidence.RankExtensionUrl).Value).Value); + Assert.DoesNotContain(evidenceExtension.Extension, extension => extension.Url == "globalRank"); + Assert.Equal(evidence.Score, ((FhirDecimal)evidenceExtension.Extension.Single(extension => extension.Url == SemanticSearchEvidence.ScoreExtensionUrl).Value).Value); + Assert.Equal(evidence.SearchParameterCanonical.OriginalString, ((FhirUri)evidenceExtension.Extension.Single(extension => extension.Url == SemanticSearchEvidence.SearchParameterExtensionUrl).Value).Value); + Assert.Equal(evidence.SourceReference, ((ResourceReference)evidenceExtension.Extension.Single(extension => extension.Url == SemanticSearchEvidence.SourceExtensionUrl).Value).Reference); + Assert.Equal(evidence.WitnessReference, ((ResourceReference)evidenceExtension.Extension.Single(extension => extension.Url == SemanticSearchEvidence.WitnessExtensionUrl).Value).Reference); + Assert.Equal(evidence.SourcePath, ((FhirString)evidenceExtension.Extension.Single(extension => extension.Url == SemanticSearchEvidence.SourcePathExtensionUrl).Value).Value); + } + private ResourceWrapper CreateResourceWrapper(ResourceElement resourceElement, HttpMethod httpMethod) { return new ResourceWrapper( diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/Expressions/Parsers/ModelInfoProviderSerialCollection.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/Expressions/Parsers/ModelInfoProviderSerialCollection.cs new file mode 100644 index 0000000000..94604f6272 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/Expressions/Parsers/ModelInfoProviderSerialCollection.cs @@ -0,0 +1,18 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.Expressions.Parsers +{ + /// + /// Serializes tests that replace the global ModelInfoProvider so they do not run in parallel with + /// tests that read it and would otherwise observe a partially configured provider. + /// + [CollectionDefinition(nameof(ModelInfoProviderSerialCollection), DisableParallelization = true)] + public sealed class ModelInfoProviderSerialCollection + { + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/Expressions/Parsers/VectorSearchExpressionParserTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/Expressions/Parsers/VectorSearchExpressionParserTests.cs new file mode 100644 index 0000000000..31b0139416 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/Expressions/Parsers/VectorSearchExpressionParserTests.cs @@ -0,0 +1,213 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions.Parsers; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; +using SearchModifierCode = Microsoft.Health.Fhir.ValueSets.SearchModifierCode; +using SearchParamType = Microsoft.Health.Fhir.ValueSets.SearchParamType; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.Expressions.Parsers +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + [Collection(nameof(ModelInfoProviderSerialCollection))] + public class VectorSearchExpressionParserTests + { + private const string SearchParameterCode = "semantic-text"; + private static readonly Uri SearchParameterCanonical = new Uri("http://example.org/fhir/SearchParameter/semantic-text"); + private readonly ISearchParameterDefinitionManager _searchParameterDefinitionManager = Substitute.For(); + private readonly IReferenceSearchValueParser _referenceSearchValueParser = Substitute.For(); + private readonly IVectorSearchParameterResolver _vectorSearchParameterResolver = Substitute.For(); + private readonly SearchParameterExpressionParser _parser; + + public VectorSearchExpressionParserTests() + { + _parser = new SearchParameterExpressionParser(_referenceSearchValueParser, _vectorSearchParameterResolver); + ModelInfoProvider.SetProvider(MockModelInfoProviderBuilder.Create(FhirSpecification.R4).Build()); + } + + [Fact] + public void GivenVectorResolverIsRegistered_WhenParserIsResolved_ThenResolverAwareConstructorIsUsed() + { + SearchParameterInfo searchParameter = CreateEnabledVectorSearchParameter(); + var services = new ServiceCollection(); + services.AddSingleton(_referenceSearchValueParser); + services.AddSingleton(_vectorSearchParameterResolver); + services.AddSingleton(); + using ServiceProvider serviceProvider = services.BuildServiceProvider(); + ISearchParameterExpressionParser parser = serviceProvider.GetRequiredService(); + + VectorSearchExpression expression = Assert.IsType(parser.Parse(searchParameter, null, "breathing difficulty")); + + Assert.Same(searchParameter, expression.Parameter); + _vectorSearchParameterResolver.Received(1).GetSearchParameter(SearchParameterCanonical); + } + + [Fact] + public void GivenStandardFhirVectorQuery_WhenParsed_ThenVectorExpressionIsCreated() + { + const string resourceType = "Resource"; + const string queryText = "breathing difficulty overnight"; + SearchParameterInfo searchParameter = CreateEnabledVectorSearchParameter(); + _searchParameterDefinitionManager.GetSearchParameter(resourceType, SearchParameterCode).Returns(searchParameter); + var expressionParser = new ExpressionParser(() => _searchParameterDefinitionManager, _parser); + + VectorSearchExpression expression = Assert.IsType( + expressionParser.Parse(new[] { resourceType }, SearchParameterCode, queryText)); + + Assert.Same(searchParameter, expression.Parameter); + Assert.Equal(queryText, expression.QueryText); + } + + [Fact] + public void GivenReverseChainedVectorQuery_WhenParsed_ThenVectorLeafAndRelationshipArePreserved() + { + const string queryText = "breathing difficulty overnight"; + SearchParameterInfo vectorSearchParameter = CreateEnabledVectorSearchParameter(); + var referenceSearchParameter = new SearchParameterInfo( + name: "subject", + code: "subject", + searchParamType: SearchParamType.Reference, + url: new Uri("http://hl7.org/fhir/SearchParameter/Observation-subject"), + targetResourceTypes: new[] { KnownResourceTypes.Patient }); + _searchParameterDefinitionManager.GetSearchParameter(KnownResourceTypes.Observation, "subject").Returns(referenceSearchParameter); + _searchParameterDefinitionManager.GetSearchParameter(KnownResourceTypes.Observation, SearchParameterCode).Returns(vectorSearchParameter); + var expressionParser = new ExpressionParser(() => _searchParameterDefinitionManager, _parser); + + ChainedExpression expression = Assert.IsType(expressionParser.Parse( + new[] { KnownResourceTypes.Patient }, + "_has:Observation:subject:semantic-text", + queryText)); + + Assert.True(expression.Reversed); + Assert.Equal(new[] { KnownResourceTypes.Observation }, expression.ResourceTypes); + Assert.Equal(new[] { KnownResourceTypes.Patient }, expression.TargetResourceTypes); + Assert.Same(referenceSearchParameter, expression.ReferenceSearchParameter); + VectorSearchExpression vectorExpression = Assert.IsType(expression.Expression); + Assert.Same(vectorSearchParameter, vectorExpression.Parameter); + Assert.Equal(queryText, vectorExpression.QueryText); + } + + [Fact] + public void GivenForwardChainedVectorQuery_WhenParsed_ThenVectorLeafAndRelationshipArePreserved() + { + const string queryText = "mobility concerns"; + SearchParameterInfo vectorSearchParameter = CreateEnabledVectorSearchParameter(); + var referenceSearchParameter = new SearchParameterInfo( + name: "subject", + code: "subject", + searchParamType: SearchParamType.Reference, + url: new Uri("http://hl7.org/fhir/SearchParameter/Observation-subject"), + targetResourceTypes: new[] { KnownResourceTypes.Patient }); + _searchParameterDefinitionManager.GetSearchParameter(KnownResourceTypes.Observation, "subject").Returns(referenceSearchParameter); + _searchParameterDefinitionManager.GetSearchParameter(KnownResourceTypes.Patient, SearchParameterCode).Returns(vectorSearchParameter); + var expressionParser = new ExpressionParser(() => _searchParameterDefinitionManager, _parser); + + ChainedExpression expression = Assert.IsType(expressionParser.Parse( + new[] { KnownResourceTypes.Observation }, + "subject:Patient.semantic-text", + queryText)); + + Assert.False(expression.Reversed); + Assert.Equal(new[] { KnownResourceTypes.Observation }, expression.ResourceTypes); + Assert.Equal(new[] { KnownResourceTypes.Patient }, expression.TargetResourceTypes); + Assert.Same(referenceSearchParameter, expression.ReferenceSearchParameter); + VectorSearchExpression vectorExpression = Assert.IsType(expression.Expression); + Assert.Same(vectorSearchParameter, vectorExpression.Parameter); + Assert.Equal(queryText, vectorExpression.QueryText); + } + + [Fact] + public void GivenVectorSearchParameter_WhenParsed_ThenVectorExpressionIsCreated() + { + const string queryText = "breathing difficulty overnight"; + SearchParameterInfo searchParameter = CreateEnabledVectorSearchParameter(); + + VectorSearchExpression expression = Assert.IsType(_parser.Parse(searchParameter, null, queryText)); + + Assert.Same(searchParameter, expression.Parameter); + Assert.Equal(queryText, expression.QueryText); + } + + [Fact] + public void GivenVectorQueryContainingComma_WhenParsed_ThenQueryRemainsSingleValue() + { + const string queryText = "asthma, overnight changes"; + + VectorSearchExpression expression = Assert.IsType(_parser.Parse(CreateEnabledVectorSearchParameter(), null, queryText)); + + Assert.Equal(queryText, expression.QueryText); + } + + [Theory] + [InlineData(SearchModifierCode.Contains)] + [InlineData(SearchModifierCode.Exact)] + [InlineData(SearchModifierCode.Missing)] + public void GivenVectorSearchParameterWithModifier_WhenParsed_ThenInvalidSearchOperationExceptionIsThrown(SearchModifierCode modifierCode) + { + var modifier = new SearchModifier(modifierCode); + + Assert.Throws(() => _parser.Parse(CreateVectorSearchParameter(), modifier, "breathing difficulty")); + } + + [Fact] + public void GivenVectorExpression_WhenRendered_ThenQueryTextIsNotExposed() + { + const string queryText = "sensitive clinical text"; + VectorSearchExpression expression = Assert.IsType(_parser.Parse(CreateEnabledVectorSearchParameter(), null, queryText)); + + Assert.DoesNotContain(queryText, expression.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void GivenVectorSearchIsNotRegistered_WhenParsed_ThenSearchParameterNotSupportedExceptionIsThrown() + { + var parser = new SearchParameterExpressionParser(_referenceSearchValueParser); + + Assert.Throws(() => parser.Parse(CreateVectorSearchParameter(), null, "breathing difficulty")); + } + + [Fact] + public void GivenVectorSearchParameterIsNotEnabled_WhenParsed_ThenSearchParameterNotSupportedExceptionIsThrown() + { + SearchParameterInfo searchParameter = CreateVectorSearchParameter(); + _vectorSearchParameterResolver.GetSearchParameter(searchParameter.Url) + .Returns(_ => throw new SearchParameterNotSupportedException(searchParameter.Url)); + + Assert.Throws(() => _parser.Parse(searchParameter, null, "breathing difficulty")); + } + + private SearchParameterInfo CreateEnabledVectorSearchParameter() + { + SearchParameterInfo searchParameter = CreateVectorSearchParameter(); + _vectorSearchParameterResolver.GetSearchParameter(searchParameter.Url).Returns(searchParameter); + return searchParameter; + } + + private static SearchParameterInfo CreateVectorSearchParameter() + { + return new SearchParameterInfo( + SearchParameterCode, + SearchParameterCode, + SearchParamType.Special, + SearchParameterCanonical, + expression: "Resource.text.div", + baseResourceTypes: new[] { "Resource" }, + vectorConfig: new VectorSearchParameterConfig(), + definitionStatus: "active"); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs index bb1c008809..a5019d86fe 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs @@ -521,6 +521,51 @@ public void GivenSearchWithSupportedSortValue_WhenCreated_ThenSearchParamShouldB Assert.Equal((_lastUpdatedSearchParameterInfo, sortOrder), Assert.Single(options.Sort)); } + [Fact] + public void GivenSemanticSearchWithScoreSort_WhenCreated_ThenScoreSortIsAccepted() + { + const string semanticParameterName = "semantic-text"; + var vectorParameter = new SearchParameterInfo(semanticParameterName, semanticParameterName, Microsoft.Health.Fhir.ValueSets.SearchParamType.Special); + _expressionParser.Parse(Arg.Any(), semanticParameterName, "fracture").Returns(new VectorSearchExpression(vectorParameter, "fracture")); + _sortingValidator.ValidateSorting(default, out Arg.Any>()).ReturnsForAnyArgs(true); + var queryParameters = new[] + { + Tuple.Create(semanticParameterName, "fracture"), + Tuple.Create(KnownQueryParameterNames.Sort, SearchParameterNames.Score), + }; + + SearchOptions options = CreateSearchOptions(resourceType: "Patient", queryParameters: queryParameters); + + Assert.Equal((SearchParameterInfo.ScoreSearchParameter, SortOrder.Ascending), Assert.Single(options.Sort)); + Assert.DoesNotContain(_defaultFhirRequestContext.BundleIssues, issue => issue.Diagnostics.Contains(SearchParameterNames.Score, StringComparison.Ordinal)); + } + + [Theory] + [InlineData(false, "_score")] + [InlineData(true, "-_score")] + public void GivenUnsupportedScoreSort_WhenCreated_ThenScoreSortIsRejected(bool isSemanticSearch, string scoreSort) + { + const string semanticParameterName = "semantic-text"; + var vectorParameter = new SearchParameterInfo(semanticParameterName, semanticParameterName, Microsoft.Health.Fhir.ValueSets.SearchParamType.Special); + if (isSemanticSearch) + { + _expressionParser.Parse(Arg.Any(), semanticParameterName, "fracture").Returns(new VectorSearchExpression(vectorParameter, "fracture")); + } + + var queryParameters = new List>(); + if (isSemanticSearch) + { + queryParameters.Add(Tuple.Create(semanticParameterName, "fracture")); + } + + queryParameters.Add(Tuple.Create(KnownQueryParameterNames.Sort, scoreSort)); + + SearchOptions options = CreateSearchOptions(resourceType: "Patient", queryParameters: queryParameters); + + Assert.Empty(options.Sort); + Assert.Contains(_defaultFhirRequestContext.BundleIssues, issue => issue.Diagnostics.Contains(SearchParameterNames.Score, StringComparison.Ordinal)); + } + [Fact] public void GivenSearchWithAnInvalidSortValue_WhenCreated_ThenAnOperationOutcomeIssueIsCreated() { diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs index a1b838eeba..662b217644 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchConverterForAllSearchTypes.cs @@ -90,7 +90,7 @@ public async Task ListAllUnsupportedTypes() { foreach (SearchParameterInfo parameterInfo in searchParameterRow.parameters) { - if (parameterInfo.Code != "_type") + if (parameterInfo.Code != "_type" && parameterInfo.VectorConfig == null) { var converters = await GetConvertsForSearchParameters(searchParameterRow.resourceType, parameterInfo); if (converters.All(x => x.hasConverter == false)) @@ -116,7 +116,7 @@ public async Task ListAllUnsupportedTypes() var systemUnsupported = new UnsupportedSearchParameters(); foreach (var searchParameter in resourceAndSearchParameters.SelectMany(x => x.parameters)) { - if (searchParameter.Code == "_type") + if (searchParameter.Code == "_type" || searchParameter.VectorConfig != null) { continue; } @@ -193,7 +193,7 @@ public static IEnumerable GetAllSearchParameters() foreach ((string resourceType, IEnumerable parameters) row in values) { - yield return new object[] { row.resourceType, row.parameters.Where(x => x.Code != "_type" && x.IsSupported) }; + yield return new object[] { row.resourceType, row.parameters.Where(x => x.Code != "_type" && x.IsSupported && x.VectorConfig == null) }; } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs index 969ab8d328..ac635ea78b 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterSupportResolverTests.cs @@ -45,6 +45,27 @@ public void GivenASupportedSearchParameter_WhenResolvingSupport_ThenTrueIsReturn Assert.False(supported.IsPartiallySupported); } + [Fact] + public void GivenAVectorSearchParameterUsingToString_WhenResolvingSupport_ThenTrueIsReturned() + { + var searchParameter = new SearchParameterInfo( + "DocumentReferenceSemanticText", + "semantic-text", + SearchParamType.Special, + new Uri("https://example.org/fhir/SearchParameter/document-reference-semantic-text"), + expression: "DocumentReference.content.attachment.url.toString()", + baseResourceTypes: new[] { "DocumentReference" }, + vectorConfig: new VectorSearchParameterConfig + { + SourceStrategy = VectorTextSourceStrategy.LocalBinaryReference, + }); + + var supported = _resolver.IsSearchParameterSupported(searchParameter); + + Assert.True(supported.Supported); + Assert.False(supported.IsPartiallySupported); + } + [Fact] public void GivenAnUnsupportedSearchParameter_WhenResolvingSupport_ThenFalseIsReturned() { diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchResourceHandlerTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchResourceHandlerTests.cs index b193028ad3..67654ba481 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchResourceHandlerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchResourceHandlerTests.cs @@ -13,6 +13,7 @@ using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Filters; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Features.Security; using Microsoft.Health.Fhir.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Messages.Search; @@ -31,16 +32,20 @@ public class SearchResourceHandlerTests { private readonly ISearchService _searchService = Substitute.For(); private readonly IBundleFactory _bundleFactory = Substitute.For(); + private readonly ISemanticSearchEvidenceFilter _semanticSearchEvidenceFilter = Substitute.For(); private readonly SearchResourceHandler _searchResourceHandler; public SearchResourceHandlerTests() { + _semanticSearchEvidenceFilter.FilterAsync(Arg.Any(), Arg.Any()) + .Returns(callInfo => callInfo.Arg()); _searchResourceHandler = new SearchResourceHandler( _searchService, _bundleFactory, DisabledFhirAuthorizationService.Instance, - new DataResourceFilter(MissingDataFilterCriteria.Default)); + new DataResourceFilter(MissingDataFilterCriteria.Default), + _semanticSearchEvidenceFilter); } [Fact] @@ -62,6 +67,23 @@ public async Task GivenASearchResourceRequest_WhenHandled_ThenABundleShouldBeRet Assert.Equal(expectedBundle, actualResponse.Bundle); } + [Fact] + public async Task GivenAFilteredSemanticSearchResult_WhenHandled_ThenBundleUsesFilteredResult() + { + var request = new SearchResourceRequest("Observation", null); + var unfilteredResult = new SearchResult(Enumerable.Empty(), null, null, Array.Empty>()); + SearchResult filteredResult = SearchResult.Empty(); + var expectedBundle = new Bundle().ToResourceElement(); + _searchService.SearchAsync(request.ResourceType, request.Queries, CancellationToken.None).Returns(unfilteredResult); + _semanticSearchEvidenceFilter.FilterAsync(unfilteredResult, CancellationToken.None).Returns(filteredResult); + _bundleFactory.CreateSearchBundle(filteredResult).Returns(expectedBundle); + + SearchResourceResponse response = await _searchResourceHandler.HandleAsync(request, CancellationToken.None); + + Assert.Equal(expectedBundle, response.Bundle); + _bundleFactory.Received(1).CreateSearchBundle(filteredResult); + } + [Fact] public async Task GivenASearchResourceRequest_WhenUserHasSearchPermission_ThenSearchSucceeds() { @@ -70,7 +92,8 @@ public async Task GivenASearchResourceRequest_WhenUserHasSearchPermission_ThenSe _searchService, _bundleFactory, authorizationService, - new DataResourceFilter(MissingDataFilterCriteria.Default)); + new DataResourceFilter(MissingDataFilterCriteria.Default), + _semanticSearchEvidenceFilter); var request = new SearchResourceRequest("Patient", null); var searchResult = new SearchResult(Enumerable.Empty(), null, null, new Tuple[0]); @@ -103,7 +126,8 @@ public async Task GivenASearchResourceRequest_WhenUserHasOnlyReadPermission_Then _searchService, _bundleFactory, authorizationService, - new DataResourceFilter(MissingDataFilterCriteria.Default)); + new DataResourceFilter(MissingDataFilterCriteria.Default), + _semanticSearchEvidenceFilter); var request = new SearchResourceRequest("Patient", null); @@ -119,7 +143,8 @@ public async Task GivenASearchResourceRequest_WhenUserHasReadAndSearchPermission _searchService, _bundleFactory, authorizationService, - new DataResourceFilter(MissingDataFilterCriteria.Default)); + new DataResourceFilter(MissingDataFilterCriteria.Default), + _semanticSearchEvidenceFilter); var request = new SearchResourceRequest("Patient", null); var searchResult = new SearchResult(Enumerable.Empty(), null, null, new Tuple[0]); @@ -147,7 +172,8 @@ public async Task GivenASearchResourceRequest_WhenUserHasNoPermissions_ThenUnaut _searchService, _bundleFactory, authorizationService, - new DataResourceFilter(MissingDataFilterCriteria.Default)); + new DataResourceFilter(MissingDataFilterCriteria.Default), + _semanticSearchEvidenceFilter); var request = new SearchResourceRequest("Patient", null); diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SemanticSearch/SemanticSearchHandlerTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SemanticSearch/SemanticSearchHandlerTests.cs new file mode 100644 index 0000000000..aec9f26952 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SemanticSearch/SemanticSearchHandlerTests.cs @@ -0,0 +1,304 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Hl7.Fhir.Model; +using Hl7.Fhir.Serialization; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Filters; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Features.Security.Authorization; +using Microsoft.Health.Fhir.Core.Messages.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; +using CompartmentType = Microsoft.Health.Fhir.ValueSets.CompartmentType; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public class SemanticSearchHandlerTests + { + private static readonly HashSet VectorResourceTypes = new HashSet(StringComparer.Ordinal) + { + ResourceType.DocumentReference.ToString(), + ResourceType.Observation.ToString(), + ResourceType.DiagnosticReport.ToString(), + }; + + private readonly ISearchService _searchService = Substitute.For(); + private readonly IDocumentReferenceSemanticSearch _semanticSearch = Substitute.For(); + private readonly IDataResourceFilter _dataResourceFilter = Substitute.For(); + private readonly ISemanticSearchEvidenceFilter _semanticSearchEvidenceFilter = Substitute.For(); + private readonly ICompartmentDefinitionManager _compartmentDefinitionManager = Substitute.For(); + private readonly IVectorSearchParameterResolver _searchParameterResolver = Substitute.For(); + + public SemanticSearchHandlerTests() + { + _semanticSearchEvidenceFilter.FilterAsync(Arg.Any(), Arg.Any()) + .Returns(callInfo => callInfo.Arg()); + _compartmentDefinitionManager.TryGetResourceTypes(CompartmentType.Patient, out Arg.Any>()) + .Returns(callInfo => + { + callInfo[1] = new HashSet(VectorResourceTypes, StringComparer.Ordinal) { ResourceType.Condition.ToString() }; + return true; + }); + _searchParameterResolver.GetSearchParameters(Arg.Any()) + .Returns(callInfo => VectorResourceTypes.Contains(callInfo.Arg()) + ? new[] { new SearchParameterInfo("semantic-text", "semantic-text") } + : Array.Empty()); + } + + [Fact] + public async Task GivenPatientSemanticSearch_WhenHandled_ThenAllSupportedResourceTypesAreGloballyRanked() + { + const string patientId = "123"; + ResourceWrapper documentReference = CreateResourceWrapper(new DocumentReference { Id = "document-reference" }, 101); + ResourceWrapper observation = CreateResourceWrapper(new Observation { Id = "observation" }, 102); + ResourceWrapper diagnosticReport = CreateResourceWrapper(new DiagnosticReport { Id = "diagnostic-report" }, 103); + + _searchService.SearchCompartmentAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any>>(), + CancellationToken.None) + .Returns(CreateSearchResult(documentReference, observation, diagnosticReport)); + _dataResourceFilter.Filter(Arg.Any()).Returns(callInfo => callInfo.Arg()); + _semanticSearch.SearchAsync( + "breathing difficulty", + Arg.Any>(), + 3, + CancellationToken.None) + .Returns(new[] + { + CreateVectorResult(observation, 0.95f), + CreateVectorResult(documentReference, 0.85f), + CreateVectorResult(diagnosticReport, 0.75f), + }); + + var handler = new SemanticSearchHandler( + _searchService, + _semanticSearch, + DisabledFhirAuthorizationService.Instance, + _dataResourceFilter, + _semanticSearchEvidenceFilter, + _compartmentDefinitionManager, + _searchParameterResolver, + Deserializers.ResourceDeserializer, + Options.Create(new VectorSearchConfiguration())); + + SemanticSearchResponse response = await handler.HandleAsync( + new SemanticSearchRequest("breathing difficulty", patientId, 3), + CancellationToken.None); + + Bundle bundle = response.Bundle.ToPoco(); + Assert.Equal(3, bundle.Total); + Assert.Collection( + bundle.Entry, + entry => Assert.IsType(entry.Resource), + entry => Assert.IsType(entry.Resource), + entry => Assert.IsType(entry.Resource)); + Assert.Equal(new decimal?[] { 0.95m, 0.85m, 0.75m }, bundle.Entry.Select(entry => entry.Search.Score).ToArray()); + int expectedRank = 1; + foreach (Bundle.EntryComponent entry in bundle.Entry) + { + Extension evidence = Assert.Single(entry.Search.Extension, extension => extension.Url == SemanticSearchEvidence.ExtensionUrl); + Assert.Equal("Matched passage", ((FhirString)evidence.Extension.Single(extension => extension.Url == SemanticSearchEvidence.TextExtensionUrl).Value).Value); + Assert.Equal(expectedRank++, ((PositiveInt)evidence.Extension.Single(extension => extension.Url == SemanticSearchEvidence.RankExtensionUrl).Value).Value); + Assert.DoesNotContain(evidence.Extension, nestedExtension => nestedExtension.Url == "globalRank"); + Assert.Equal($"{entry.Resource.TypeName}.text", ((FhirString)evidence.Extension.Single(extension => extension.Url == SemanticSearchEvidence.SourcePathExtensionUrl).Value).Value); + Assert.Equal( + $"{entry.Resource.TypeName}/{entry.Resource.Id}", + ((ResourceReference)evidence.Extension.Single(extension => extension.Url == SemanticSearchEvidence.SourceExtensionUrl).Value).Reference); + } + + await _searchService.Received(1).SearchCompartmentAsync( + CompartmentType.Patient.ToString(), + patientId, + null, + Arg.Is>>((IReadOnlyList> parameters) => + parameters.Contains(Tuple.Create(SearchParameterNames.ResourceType, "DiagnosticReport,DocumentReference,Observation")) && + parameters.Any(parameter => parameter.Item1 == "_count")), + CancellationToken.None); + + await _semanticSearch.Received(1).SearchAsync( + "breathing difficulty", + Arg.Is>(candidates => candidates.Count == 3), + 3, + CancellationToken.None); + await _semanticSearchEvidenceFilter.Received(1).FilterAsync(Arg.Any(), CancellationToken.None); + } + + [Fact] + public async Task GivenSelectedResourceTypes_WhenHandled_ThenOnlySelectedTypesAreSearched() + { + var emptyResult = new SearchResult( + Array.Empty(), + continuationToken: null, + sortOrder: null, + unsupportedSearchParameters: Array.Empty>()); + _searchService.SearchCompartmentAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any>>(), + CancellationToken.None) + .Returns(emptyResult); + _dataResourceFilter.Filter(emptyResult).Returns(emptyResult); + _semanticSearch.SearchAsync( + Arg.Any(), + Arg.Any>(), + Arg.Any(), + CancellationToken.None) + .Returns(Array.Empty()); + var handler = new SemanticSearchHandler( + _searchService, + _semanticSearch, + DisabledFhirAuthorizationService.Instance, + _dataResourceFilter, + _semanticSearchEvidenceFilter, + _compartmentDefinitionManager, + _searchParameterResolver, + Deserializers.ResourceDeserializer, + Options.Create(new VectorSearchConfiguration())); + + await handler.HandleAsync( + new SemanticSearchRequest("breathing difficulty", "123", 3, new[] { "Observation" }), + CancellationToken.None); + + await _searchService.Received(1).SearchCompartmentAsync( + CompartmentType.Patient.ToString(), + "123", + null, + Arg.Is>>((IReadOnlyList> parameters) => + parameters.Contains(Tuple.Create(SearchParameterNames.ResourceType, "Observation"))), + CancellationToken.None); + } + + [Fact] + public async Task GivenRequestedTypeWithoutVectorSearchParameter_WhenHandled_ThenRequestIsRejected() + { + SemanticSearchHandler handler = CreateHandler(); + + await Assert.ThrowsAsync(() => handler.HandleAsync( + new SemanticSearchRequest("breathing difficulty", "123", 3, new[] { "Condition" }), + CancellationToken.None)); + + await _searchService.DidNotReceive().SearchCompartmentAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any>>(), + Arg.Any()); + } + + [Fact] + public async Task GivenEvidenceFilterRemovesResult_WhenHandled_ThenResultIsNotReturned() + { + ResourceWrapper observation = CreateResourceWrapper(new Observation { Id = "observation" }, 102); + SearchResult candidateResult = CreateSearchResult(observation); + _searchService.SearchCompartmentAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any>>(), + CancellationToken.None) + .Returns(candidateResult); + _dataResourceFilter.Filter(candidateResult).Returns(candidateResult); + _semanticSearch.SearchAsync( + Arg.Any(), + Arg.Any>(), + Arg.Any(), + CancellationToken.None) + .Returns(new[] { CreateVectorResult(observation, 0.95f) }); + _semanticSearchEvidenceFilter.FilterAsync(Arg.Any(), CancellationToken.None) + .Returns(SearchResult.Empty()); + var handler = new SemanticSearchHandler( + _searchService, + _semanticSearch, + DisabledFhirAuthorizationService.Instance, + _dataResourceFilter, + _semanticSearchEvidenceFilter, + _compartmentDefinitionManager, + _searchParameterResolver, + Deserializers.ResourceDeserializer, + Options.Create(new VectorSearchConfiguration())); + + SemanticSearchResponse response = await handler.HandleAsync( + new SemanticSearchRequest("breathing difficulty", "123", 3, new[] { "Observation" }), + CancellationToken.None); + + Bundle bundle = response.Bundle.ToPoco(); + Assert.Equal(0, bundle.Total); + Assert.Empty(bundle.Entry); + } + + private SemanticSearchHandler CreateHandler() + { + return new SemanticSearchHandler( + _searchService, + _semanticSearch, + DisabledFhirAuthorizationService.Instance, + _dataResourceFilter, + _semanticSearchEvidenceFilter, + _compartmentDefinitionManager, + _searchParameterResolver, + Deserializers.ResourceDeserializer, + Options.Create(new VectorSearchConfiguration())); + } + + private static SearchResult CreateSearchResult(params ResourceWrapper[] resources) + { + return new SearchResult( + resources.Select(resource => new SearchResultEntry(resource)), + continuationToken: null, + sortOrder: null, + unsupportedSearchParameters: Array.Empty>()); + } + + private static VectorSearchResult CreateVectorResult(ResourceWrapper resource, float score) + { + var evidence = new SemanticSearchEvidence( + "Matched passage", + chunkOrdinal: 0, + score: (decimal)score, + new Uri($"https://example.org/fhir/SearchParameter/{resource.ResourceTypeName}-semantic"), + $"{resource.ResourceTypeName}/{resource.ResourceId}", + $"{resource.ResourceTypeName}.text"); + return new VectorSearchResult(resource.ResourceTypeName, resource.ResourceSurrogateId, score, evidence); + } + + private static ResourceWrapper CreateResourceWrapper(Resource resource, long resourceSurrogateId) + { + resource.Meta = new Meta { VersionId = "1" }; + var serializer = new FhirJsonSerializer(); + return new ResourceWrapper( + resource.ToResourceElement(), + new RawResource(serializer.SerializeToString(resource), FhirResourceFormat.Json, isMetaSet: true), + new ResourceRequest(HttpMethod.Post, "http://test/resource"), + deleted: false, + searchIndices: null, + compartmentIndices: null, + lastModifiedClaims: null, + resourceSurrogateId: resourceSurrogateId); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems index 5c4f1e776a..73794f8f3f 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems @@ -130,10 +130,13 @@ + + + diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Create/CreateResourceHandler.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Create/CreateResourceHandler.cs index aaaa2cb8ca..693b94f148 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Create/CreateResourceHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Create/CreateResourceHandler.cs @@ -14,6 +14,7 @@ using Microsoft.Health.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Conformance; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Security; @@ -39,7 +40,6 @@ public CreateResourceHandler( : base(fhirDataStore, conformanceProvider, resourceWrapperFactory, resourceIdProvider, authorizationService) { EnsureArg.IsNotNull(referenceResolver, nameof(referenceResolver)); - _referenceResolver = referenceResolver; _referenceIdDictionary = new Dictionary(); } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Upsert/UpsertResourceHandler.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Upsert/UpsertResourceHandler.cs index 3ec8134c0a..ef084ed0b4 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Upsert/UpsertResourceHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Upsert/UpsertResourceHandler.cs @@ -16,6 +16,7 @@ using Microsoft.Health.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Conformance; using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Persistence; @@ -50,7 +51,6 @@ public UpsertResourceHandler( EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider)); EnsureArg.IsNotNull(referenceResolver, nameof(referenceResolver)); EnsureArg.IsNotNull(contextAccessor, nameof(contextAccessor)); - _referenceResolver = referenceResolver; _modelInfoProvider = modelInfoProvider; _contextAccessor = contextAccessor; diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs index 6603cd9b7d..2586beb266 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs @@ -16,6 +16,7 @@ using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Features.Telemetry; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Shared.Core.Features.Search; @@ -50,12 +51,49 @@ public ResourceElement CreateSearchBundle(SearchResult result) resource.Search = new Bundle.SearchComponent { Mode = r.SearchEntryMode == SearchEntryMode.Match ? Bundle.SearchEntryMode.Match : Bundle.SearchEntryMode.Include, + Score = r.Score, }; + foreach (SemanticSearchEvidence evidence in r.EvidenceItems) + { + resource.Search.Extension.Add(CreateSemanticEvidenceExtension(evidence)); + } + return resource; }); } + internal static Extension CreateSemanticEvidenceExtension(SemanticSearchEvidence evidence) + { + var extension = new Extension + { + Url = SemanticSearchEvidence.ExtensionUrl, + }; + + extension.Extension.Add(new Extension(SemanticSearchEvidence.TextExtensionUrl, new FhirString(evidence.Text))); + extension.Extension.Add(new Extension(SemanticSearchEvidence.ChunkOrdinalExtensionUrl, new Integer(evidence.ChunkOrdinal))); + if (evidence.Rank.HasValue) + { + extension.Extension.Add(new Extension(SemanticSearchEvidence.RankExtensionUrl, new PositiveInt(evidence.Rank.Value))); + } + + if (evidence.Score.HasValue) + { + extension.Extension.Add(new Extension(SemanticSearchEvidence.ScoreExtensionUrl, new FhirDecimal(evidence.Score.Value))); + } + + extension.Extension.Add(new Extension(SemanticSearchEvidence.SearchParameterExtensionUrl, new FhirUri(evidence.SearchParameterCanonical))); + extension.Extension.Add(new Extension(SemanticSearchEvidence.SourceExtensionUrl, new ResourceReference(evidence.SourceReference))); + if (evidence.WitnessReference != null) + { + extension.Extension.Add(new Extension(SemanticSearchEvidence.WitnessExtensionUrl, new ResourceReference(evidence.WitnessReference))); + } + + extension.Extension.Add(new Extension(SemanticSearchEvidence.SourcePathExtensionUrl, new FhirString(evidence.SourcePath))); + + return extension; + } + public ResourceElement CreateHistoryBundle(SearchResult result) { return CreateBundle(result, Bundle.BundleType.History, r => diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterToTypeResolver.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterToTypeResolver.cs index fb47782b81..2e172a85d3 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterToTypeResolver.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterToTypeResolver.cs @@ -143,6 +143,13 @@ private static EnumerableReturnType Visit(FunctionCallExpression expression, Con yield break; } + case "toString": + { + yield return new SearchParameterTypeResult(GetMapping(typeof(FhirString)), ctx.SearchParamType, null, ctx.Definition); + + yield break; + } + case "as": case "ofType": { diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs index 6866517dff..f2a15de8b4 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs @@ -543,6 +543,21 @@ public SearchOptions Create( // Only parameters that are valid for searching can also be used as sort parameter values. Therefore first check if the sort parameter values are valid as search parameters. foreach ((string, Hl7.Fhir.Rest.SortOrder) sorting in searchParams.Sort) { + if (string.Equals(sorting.Item1, SearchParameterNames.Score, StringComparison.Ordinal)) + { + if (sorting.Item2 == Hl7.Fhir.Rest.SortOrder.Ascending && ContainsVectorSearch(searchOptions.Expression)) + { + sortings.Add((SearchParameterInfo.ScoreSearchParameter, SortOrder.Ascending)); + } + else + { + sortingsValid = false; + otherSearchErrors.Add(string.Format(CultureInfo.InvariantCulture, Core.Resources.SearchSortParameterNotSupported, sorting.Item1)); + } + + continue; + } + try { SearchParameterInfo searchParameterInfo = resourceTypesString.Select(t => _searchParameterDefinitionManager.GetSearchParameter(t, sorting.Item1)).Distinct().First(); @@ -637,6 +652,11 @@ public SearchOptions Create( return searchOptions; } + private static bool ContainsVectorSearch(Expression expression) + { + return expression?.AcceptVisitor(VectorSearchPresenceVisitor.Instance, context: null) ?? false; + } + private IEnumerable ParseIncludeIterateExpressions(IList<(string query, IncludeModifier modifier)> includes, string[] typesString, bool isReversed) { return includes.Select(p => @@ -953,5 +973,17 @@ private void CheckFineGrainedAccessControl(List searchExpressions, S } } } + + private sealed class VectorSearchPresenceVisitor : DefaultExpressionVisitor + { + public static readonly VectorSearchPresenceVisitor Instance = new VectorSearchPresenceVisitor(); + + private VectorSearchPresenceVisitor() + : base((found, current) => found || current) + { + } + + public override bool VisitVectorSearch(VectorSearchExpression expression, object context) => true; + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchResourceHandler.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchResourceHandler.cs index baf14e95a6..6db01cf893 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchResourceHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchResourceHandler.cs @@ -9,6 +9,7 @@ using Medino; using Microsoft.Health.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Features.Security; using Microsoft.Health.Fhir.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Messages.Search; @@ -25,6 +26,7 @@ public class SearchResourceHandler : IRequestHandler _authorizationService; private readonly IDataResourceFilter _dataResourceFilter; + private readonly ISemanticSearchEvidenceFilter _semanticSearchEvidenceFilter; /// /// Initializes a new instance of the class. @@ -33,17 +35,25 @@ public class SearchResourceHandler : IRequestHandlerThe bundle factory. /// The authorization service. /// The search result filter. - public SearchResourceHandler(ISearchService searchService, IBundleFactory bundleFactory, IAuthorizationService authorizationService, IDataResourceFilter dataResourceFilter) + /// The semantic evidence authorization filter. + public SearchResourceHandler( + ISearchService searchService, + IBundleFactory bundleFactory, + IAuthorizationService authorizationService, + IDataResourceFilter dataResourceFilter, + ISemanticSearchEvidenceFilter semanticSearchEvidenceFilter) { EnsureArg.IsNotNull(searchService, nameof(searchService)); EnsureArg.IsNotNull(bundleFactory, nameof(bundleFactory)); EnsureArg.IsNotNull(authorizationService, nameof(authorizationService)); EnsureArg.IsNotNull(dataResourceFilter, nameof(dataResourceFilter)); + EnsureArg.IsNotNull(semanticSearchEvidenceFilter, nameof(semanticSearchEvidenceFilter)); _searchService = searchService; _bundleFactory = bundleFactory; _authorizationService = authorizationService; _dataResourceFilter = dataResourceFilter; + _semanticSearchEvidenceFilter = semanticSearchEvidenceFilter; } /// @@ -68,6 +78,7 @@ await _authorizationService.CheckAccess( cancellationToken: cancellationToken, isIncludesOperation: request.IsIncludesRequest); searchResult = _dataResourceFilter.Filter(searchResult: searchResult); + searchResult = await _semanticSearchEvidenceFilter.FilterAsync(searchResult, cancellationToken); ResourceElement bundle = _bundleFactory.CreateSearchBundle(searchResult); diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SemanticSearch/SemanticSearchHandler.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SemanticSearch/SemanticSearchHandler.cs new file mode 100644 index 0000000000..4ce357934f --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SemanticSearch/SemanticSearchHandler.cs @@ -0,0 +1,170 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Hl7.Fhir.Model; +using Medino; +using Microsoft.Extensions.Options; +using Microsoft.Health.Core.Features.Security.Authorization; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Features.Security; +using Microsoft.Health.Fhir.Core.Features.Security.Authorization; +using Microsoft.Health.Fhir.Core.Messages.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using CompartmentType = Microsoft.Health.Fhir.ValueSets.CompartmentType; + +namespace Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch +{ + /// + /// Filters patient-scoped candidates with ordinary FHIR search before semantic ranking. + /// + public sealed class SemanticSearchHandler : IRequestHandler + { + private readonly ISearchService _searchService; + private readonly IDocumentReferenceSemanticSearch _semanticSearch; + private readonly IAuthorizationService _authorizationService; + private readonly IDataResourceFilter _dataResourceFilter; + private readonly ISemanticSearchEvidenceFilter _semanticSearchEvidenceFilter; + private readonly ICompartmentDefinitionManager _compartmentDefinitionManager; + private readonly IVectorSearchParameterResolver _searchParameterResolver; + private readonly ResourceDeserializer _resourceDeserializer; + private readonly VectorSearchQueryConfiguration _queryConfiguration; + + /// + /// Initializes a new instance of the class. + /// + public SemanticSearchHandler( + ISearchService searchService, + IDocumentReferenceSemanticSearch semanticSearch, + IAuthorizationService authorizationService, + IDataResourceFilter dataResourceFilter, + ISemanticSearchEvidenceFilter semanticSearchEvidenceFilter, + ICompartmentDefinitionManager compartmentDefinitionManager, + IVectorSearchParameterResolver searchParameterResolver, + ResourceDeserializer resourceDeserializer, + IOptions configuration) + { + _searchService = EnsureArg.IsNotNull(searchService, nameof(searchService)); + _semanticSearch = EnsureArg.IsNotNull(semanticSearch, nameof(semanticSearch)); + _authorizationService = EnsureArg.IsNotNull(authorizationService, nameof(authorizationService)); + _dataResourceFilter = EnsureArg.IsNotNull(dataResourceFilter, nameof(dataResourceFilter)); + _semanticSearchEvidenceFilter = EnsureArg.IsNotNull(semanticSearchEvidenceFilter, nameof(semanticSearchEvidenceFilter)); + _compartmentDefinitionManager = EnsureArg.IsNotNull(compartmentDefinitionManager, nameof(compartmentDefinitionManager)); + _searchParameterResolver = EnsureArg.IsNotNull(searchParameterResolver, nameof(searchParameterResolver)); + _resourceDeserializer = EnsureArg.IsNotNull(resourceDeserializer, nameof(resourceDeserializer)); + _queryConfiguration = EnsureArg.IsNotNull(configuration, nameof(configuration)).Value.Query; + } + + /// + public async Task HandleAsync(SemanticSearchRequest request, CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(request, nameof(request)); + await _authorizationService.CheckAccess(DataActions.Read, true, cancellationToken); + + if (!_compartmentDefinitionManager.TryGetResourceTypes(CompartmentType.Patient, out HashSet compartmentResourceTypes)) + { + throw new InvalidOperationException("The Patient compartment definition is unavailable."); + } + + var eligibleResourceTypes = compartmentResourceTypes + .Where(resourceType => _searchParameterResolver.GetSearchParameters(resourceType).Count > 0) + .ToHashSet(StringComparer.Ordinal); + string unsupportedResourceType = request.ResourceTypes.FirstOrDefault(resourceType => !eligibleResourceTypes.Contains(resourceType)); + if (unsupportedResourceType != null) + { + throw new RequestNotValidException($"Resource type '{unsupportedResourceType}' is not eligible for patient semantic search."); + } + + List selectedResourceTypes = (request.ResourceTypes.Count == 0 ? eligibleResourceTypes : request.ResourceTypes) + .OrderBy(resourceType => resourceType, StringComparer.Ordinal) + .ToList(); + if (selectedResourceTypes.Count == 0) + { + return new SemanticSearchResponse(CreateBundle(Array.Empty(), Array.Empty>()).ToResourceElement()); + } + + var searchParameters = new List> + { + Tuple.Create(SearchParameterNames.ResourceType, string.Join(',', selectedResourceTypes)), + Tuple.Create(KnownQueryParameterNames.Count, _queryConfiguration.CandidateCount.ToString(CultureInfo.InvariantCulture)), + }; + SearchResult searchResult = await _searchService.SearchCompartmentAsync( + CompartmentType.Patient.ToString(), + request.PatientId, + resourceType: null, + searchParameters, + cancellationToken); + searchResult = _dataResourceFilter.Filter(searchResult); + List candidates = searchResult.Results + .Where(result => result.SearchEntryMode == ValueSets.SearchEntryMode.Match) + .Select(result => result.Resource) + .ToList(); + + IReadOnlyList ranked = await _semanticSearch.SearchAsync( + request.Query, + candidates, + request.Count, + cancellationToken); + Dictionary<(string ResourceTypeName, long ResourceSurrogateId), ResourceWrapper> candidatesById = candidates.ToDictionary( + candidate => (candidate.ResourceTypeName, candidate.ResourceSurrogateId)); + List returnedResults = ranked + .Where(result => candidatesById.ContainsKey((result.ResourceTypeName, result.ResourceSurrogateId))) + .ToList(); + var semanticSearchResult = new SearchResult( + returnedResults.Select(result => new SearchResultEntry( + candidatesById[(result.ResourceTypeName, result.ResourceSurrogateId)], + ValueSets.SearchEntryMode.Match, + (decimal)result.Score, + evidenceItems: result.EvidenceItems)), + continuationToken: null, + sortOrder: null, + unsupportedSearchParameters: Array.Empty>()); + semanticSearchResult = await _semanticSearchEvidenceFilter.FilterAsync(semanticSearchResult, cancellationToken); + List returnedEntries = semanticSearchResult.Results.ToList(); + IReadOnlyList> rankedEvidence = SemanticSearchEvidenceRanker.AssignRanks( + returnedEntries.Select(result => result.EvidenceItems).ToList()); + + Bundle bundle = CreateBundle(returnedEntries, rankedEvidence); + + return new SemanticSearchResponse(bundle.ToResourceElement()); + } + + private Bundle CreateBundle( + IReadOnlyList returnedEntries, + IReadOnlyList> rankedEvidence) + { + return new Bundle + { + Type = Bundle.BundleType.Searchset, + Total = returnedEntries.Count, + Entry = returnedEntries + .Select((result, index) => new Bundle.EntryComponent + { + Resource = new RawResourceElement(result.Resource).ToPoco(_resourceDeserializer), + Search = new Bundle.SearchComponent + { + Mode = Bundle.SearchEntryMode.Match, + Score = result.Score, + Extension = rankedEvidence[index] + .Select(BundleFactory.CreateSemanticEvidenceExtension) + .ToList(), + }, + }) + .ToList(), + }; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Microsoft.Health.Fhir.Shared.Core.projitems b/src/Microsoft.Health.Fhir.Shared.Core/Microsoft.Health.Fhir.Shared.Core.projitems index 0dc5d3f854..3453bd7c45 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Microsoft.Health.Fhir.Shared.Core.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Core/Microsoft.Health.Fhir.Shared.Core.projitems @@ -91,6 +91,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Shared.Web/Startup.cs b/src/Microsoft.Health.Fhir.Shared.Web/Startup.cs index 7bd448710d..b800486ba3 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/Startup.cs +++ b/src/Microsoft.Health.Fhir.Shared.Web/Startup.cs @@ -8,6 +8,7 @@ using System.Diagnostics.Metrics; using System.Globalization; using System.Linq; +using Azure.Core; using Azure.Monitor.OpenTelemetry.AspNetCore; using Medino; using Microsoft.ApplicationInsights.Extensibility; @@ -18,6 +19,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.Extensions.Primitives; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Api.Features.BackgroundJobService; @@ -25,14 +27,17 @@ using Microsoft.Health.Fhir.Api.OpenIddict.Extensions; using Microsoft.Health.Fhir.Api.OpenIddict.FeatureProviders; using Microsoft.Health.Fhir.Azure; +using Microsoft.Health.Fhir.Azure.SemanticSearch; using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Features.Telemetry; using Microsoft.Health.Fhir.Core.Logging.Metrics; using Microsoft.Health.Fhir.Core.Messages.Search; using Microsoft.Health.Fhir.Core.Registration; using Microsoft.Health.Fhir.Shared.Web; +using Microsoft.Health.Fhir.SqlServer.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.JobManagement; using Microsoft.Health.SqlServer.Configs; @@ -61,10 +66,15 @@ public virtual void ConfigureServices(IServiceCollection services) { instanceId = $"{Configuration["WEBSITE_ROLE_INSTANCE_ID"]}--{Configuration["WEBSITE_INSTANCE_ID"]}--{Guid.NewGuid()}"; + VectorSearchConfiguration vectorSearchConfiguration = null; Core.Registration.IFhirServerBuilder fhirServerBuilder = services.AddFhirServer( Configuration, - fhirServerConfiguration => fhirServerConfiguration.Security.AddAuthenticationLibrary = AddAuthenticationLibrary, + fhirServerConfiguration => + { + fhirServerConfiguration.Security.AddAuthenticationLibrary = AddAuthenticationLibrary; + vectorSearchConfiguration = fhirServerConfiguration.CoreFeatures.VectorSearch; + }, mvcBuilderAction: builder => { builder.PartManager.FeatureProviders.Remove(builder.PartManager.FeatureProviders.OfType().FirstOrDefault()); @@ -83,6 +93,7 @@ public virtual void ConfigureServices(IServiceCollection services) IFhirRuntimeConfiguration runtimeConfiguration = AddRuntimeConfiguration(Configuration, fhirServerBuilder); AddDataStore(services, fhirServerBuilder, runtimeConfiguration); + AddSemanticSearch(services, runtimeConfiguration, vectorSearchConfiguration); // Set task hosting and related background service if (bool.TryParse(Configuration["TaskHosting:Enabled"], out bool taskHostingsOn) && taskHostingsOn) @@ -117,6 +128,34 @@ private void AddDataStore(IServiceCollection services, IFhirServerBuilder fhirSe } } + private void AddSemanticSearch( + IServiceCollection services, + IFhirRuntimeConfiguration runtimeConfiguration, + VectorSearchConfiguration vectorSearchConfiguration) + { + if (runtimeConfiguration is not AzureHealthDataServicesRuntimeConfiguration || + !vectorSearchConfiguration.Enabled) + { + return; + } + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddScoped(); + services.AddScoped(provider => new AzureFoundryEmbeddingClient( + provider.GetRequiredService>().Value.Embedding, + provider.GetRequiredService())); + services.AddScoped(_ => new SqlVectorStore(Configuration["SqlServer:ConnectionString"])); + services.AddSingleton(provider => new SqlEmbeddingModelRegistry( + Configuration["SqlServer:ConnectionString"], + provider.GetRequiredService>())); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + } + private static IFhirRuntimeConfiguration AddRuntimeConfiguration(IConfiguration configuration, IFhirServerBuilder fhirServerBuilder) { IFhirRuntimeConfiguration runtimeConfiguration = null; @@ -257,6 +296,14 @@ private static void AddAuthenticationLibrary(IServiceCollection services, Securi options.Authority = securityConfiguration.Authentication.Authority; options.Audience = securityConfiguration.Authentication.Audience; options.TokenValidationParameters.RoleClaimType = securityConfiguration.Authorization.RolesClaim; + + // Accept issuer with or without trailing slash (common OpenIddict variation). + string normalizedAuthority = securityConfiguration.Authentication.Authority?.TrimEnd('/'); + if (!string.IsNullOrWhiteSpace(normalizedAuthority)) + { + options.TokenValidationParameters.ValidIssuers = new[] { normalizedAuthority, normalizedAuthority + "/" }; + } + options.MapInboundClaims = false; options.RequireHttpsMetadata = true; options.Challenge = $"Bearer authorization_uri=\"{securityConfiguration.Authentication.Authority}\", resource_id=\"{securityConfiguration.Authentication.Audience}\", realm=\"{securityConfiguration.Authentication.Audience}\""; diff --git a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json index cf11c9886f..65c45957e7 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json +++ b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json @@ -41,6 +41,33 @@ "Default": "versioned", "ResourceTypeOverrides": null }, + "VectorSearch": { + "Enabled": false, + "Embedding": { + "Endpoint": null, + "DeploymentName": null, + "ModelName": "text-embedding-3-small", + "ModelVersion": null, + "Dimensions": 1536 + }, + "Indexing": { + "Mode": "Synchronous", + "ChunkSizeTokens": 800, + "ChunkOverlapTokens": 100, + "Pdf": { + "MaximumFileSizeBytes": 10485760, + "MaximumPageCount": 200, + "MaximumExtractedCharacters": 500000, + "ExtractionTimeout": "00:00:30" + } + }, + "Query": { + "DefaultCount": 10, + "MaxCount": 50, + "CandidateCount": 100, + "DistanceMetric": "cosine" + } + }, "SearchParameterCacheRefreshIntervalSeconds": 20, "SystemConformanceProviderRefreshIntervalSeconds": 60, "SystemConformanceProviderRebuildIntervalSeconds": 14400 diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Reindex/VectorSearchSourceRefreshJobTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Reindex/VectorSearchSourceRefreshJobTests.cs new file mode 100644 index 0000000000..2b3c2b2bc2 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Reindex/VectorSearchSourceRefreshJobTests.cs @@ -0,0 +1,154 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Operations.Reindex; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.SqlServer.Features.Operations; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.ValueSets; +using Microsoft.Health.JobManagement; +using Microsoft.Health.Test.Utilities; +using Newtonsoft.Json; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Operations.Reindex +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.IndexAndReindex)] + public class VectorSearchSourceRefreshJobTests + { + private readonly IVectorSearchSourceDependencyStore _dependencyStore = Substitute.For(); + private readonly IFhirDataStore _fhirDataStore = Substitute.For(); + private readonly IResourceWrapperFactory _resourceWrapperFactory = Substitute.For(); + private readonly IVectorSearchIndexer _vectorSearchIndexer = Substitute.For(); + + public VectorSearchSourceRefreshJobTests() + { + ModelInfoProvider.SetProvider( + MockModelInfoProviderBuilder.Create(FhirSpecification.R4) + .AddKnownTypes(KnownResourceTypes.DocumentReference) + .Build()); + } + + [Fact] + public async Task GivenNoDependentResources_WhenExecuted_ThenNoIndexUpdateOccurs() + { + VectorSearchSourceRefreshJob job = CreateJob(); + JobInfo jobInfo = CreateJobInfo(); + + await job.ExecuteAsync(jobInfo, CancellationToken.None); + + await _fhirDataStore.DidNotReceive().GetAsync(Arg.Any>(), Arg.Any()); + await _vectorSearchIndexer.DidNotReceive().IndexAsync(Arg.Any>(), Arg.Any()); + await _fhirDataStore.DidNotReceive().BulkUpdateSearchParameterIndicesAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task GivenDependentResources_WhenExecuted_ThenCurrentOwnersAreVectorIndexedAndPersisted() + { + var ownerKey = new ResourceKey("DocumentReference", "owner"); + ResourceWrapper owner = CreateResourceWrapper(ownerKey); + _dependencyStore.GetDependentResourceKeysAsync("Binary", "source", Arg.Any()).Returns(new[] { ownerKey }); + _fhirDataStore.GetAsync(Arg.Any>(), Arg.Any()).Returns(new[] { owner }); + VectorSearchSourceRefreshJob job = CreateJob(); + + await job.ExecuteAsync(CreateJobInfo(), CancellationToken.None); + + _resourceWrapperFactory.Received(1).Update(owner); + await _vectorSearchIndexer.Received(1).IndexAsync( + Arg.Is>(resources => resources.Count == 1 && resources.Contains(owner)), + CancellationToken.None); + await _fhirDataStore.Received(1).BulkUpdateSearchParameterIndicesAsync( + Arg.Is>(resources => resources.Count == 1 && resources.Contains(owner)), + CancellationToken.None); + } + + [Fact] + public async Task GivenDependentOwnerWasDeleted_WhenExecuted_ThenNoIndexUpdateOccurs() + { + var ownerKey = new ResourceKey("DocumentReference", "owner"); + _dependencyStore.GetDependentResourceKeysAsync("Binary", "source", Arg.Any()).Returns(new[] { ownerKey }); + _fhirDataStore.GetAsync(Arg.Any>(), Arg.Any()).Returns(Array.Empty()); + VectorSearchSourceRefreshJob job = CreateJob(); + + await job.ExecuteAsync(CreateJobInfo(), CancellationToken.None); + + await _vectorSearchIndexer.DidNotReceive().IndexAsync(Arg.Any>(), Arg.Any()); + await _fhirDataStore.DidNotReceive().BulkUpdateSearchParameterIndicesAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task GivenOwnerVersionConflict_WhenExecuted_ThenJobSoftFailsForRetry() + { + var ownerKey = new ResourceKey("DocumentReference", "owner"); + ResourceWrapper owner = CreateResourceWrapper(ownerKey); + _dependencyStore.GetDependentResourceKeysAsync("Binary", "source", Arg.Any()).Returns(new[] { ownerKey }); + _fhirDataStore.GetAsync(Arg.Any>(), Arg.Any()).Returns(new[] { owner }); + _fhirDataStore.BulkUpdateSearchParameterIndicesAsync(Arg.Any>(), Arg.Any()) + .Returns(Task.FromException(new PreconditionFailedException("conflict"))); + VectorSearchSourceRefreshJob job = CreateJob(); + + await Assert.ThrowsAsync(() => job.ExecuteAsync(CreateJobInfo(), CancellationToken.None)); + } + + private VectorSearchSourceRefreshJob CreateJob() + { + IScoped scope = Substitute.For>(); + scope.Value.Returns(_fhirDataStore); + + return new VectorSearchSourceRefreshJob( + _dependencyStore, + () => scope, + _resourceWrapperFactory, + NullLogger.Instance, + _vectorSearchIndexer); + } + + private static JobInfo CreateJobInfo() + { + return new JobInfo + { + Id = 1, + GroupId = 1, + QueueType = (byte)QueueType.Reindex, + Definition = JsonConvert.SerializeObject(new VectorSearchSourceRefreshJobDefinition + { + TypeId = (int)JobType.VectorSearchSourceRefresh, + SourceResourceType = "Binary", + SourceResourceId = "source", + SourceResourceVersion = "2", + }), + }; + } + + private static ResourceWrapper CreateResourceWrapper(ResourceKey key) + { + return new ResourceWrapper( + key.Id, + "1", + key.ResourceType, + new RawResource("{}", FhirResourceFormat.Json, isMetaSet: false), + null, + DateTimeOffset.MinValue, + deleted: false, + searchIndices: null, + compartmentIndices: null, + lastModifiedClaims: null); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/ContinuationTokenTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/ContinuationTokenTests.cs index f508cd1e1d..bbb6ece81e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/ContinuationTokenTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/ContinuationTokenTests.cs @@ -64,6 +64,26 @@ public void GivenThreeElementArray_WhenCreatingToken_ThenSortValueIsAccessible() Assert.Equal("sortValue", continuationToken.SortValue); } + [Theory] + [InlineData("[\"0.125\",103,12345]", true)] + [InlineData("[\"NaN\",103,12345]", false)] + [InlineData("[\"0.125\",0,12345]", false)] + [InlineData("[\"0.125\",103]", false)] + public void GivenSemanticCursor_WhenDecoded_ThenValidatesShapeAndValues(string json, bool expected) + { + ContinuationToken continuationToken = ContinuationToken.FromString(json); + + bool result = continuationToken.TryGetSemanticCursor(out double distance, out short resourceTypeId, out long resourceSurrogateId); + + Assert.Equal(expected, result); + if (expected) + { + Assert.Equal(0.125, distance); + Assert.Equal((short)103, resourceTypeId); + Assert.Equal(12345L, resourceSurrogateId); + } + } + [Fact] public void GivenResourceTypeIdAsLong_WhenAccessing_ThenConvertsToShort() { diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/RemoveVectorSearchRewriterTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/RemoveVectorSearchRewriterTests.cs new file mode 100644 index 0000000000..d906edafca --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/RemoveVectorSearchRewriterTests.cs @@ -0,0 +1,73 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.ValueSets; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public class RemoveVectorSearchRewriterTests + { + public RemoveVectorSearchRewriterTests() + { + ModelInfoProvider.SetProvider(MockModelInfoProviderBuilder.Create(FhirSpecification.R4).Build()); + } + + [Fact] + public void GivenVectorAndStructuredExpressions_WhenVisited_ThenOnlyVectorExpressionIsRemoved() + { + var searchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/semantic-text")); + var vectorExpression = new VectorSearchExpression(searchParameter, "query text"); + BinaryExpression structuredExpression = Expression.Equals(FieldName.Number, null, 1); + Expression structuredConjunction = Expression.And(structuredExpression, structuredExpression); + + Assert.Null(vectorExpression.AcceptVisitor(RemoveVectorSearchRewriter.Instance)); + Assert.Same(structuredExpression, structuredExpression.AcceptVisitor(RemoveVectorSearchRewriter.Instance)); + Assert.Same(structuredExpression, Expression.And(vectorExpression, structuredExpression).AcceptVisitor(RemoveVectorSearchRewriter.Instance)); + Assert.Same(structuredExpression, Expression.And(structuredExpression, vectorExpression).AcceptVisitor(RemoveVectorSearchRewriter.Instance)); + Assert.Equal( + structuredConjunction.ToString(), + Expression.And(structuredExpression, vectorExpression, structuredExpression).AcceptVisitor(RemoveVectorSearchRewriter.Instance).ToString()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void GivenSemanticOnlyChain_WhenVisited_ThenEntireChainIsRemoved(bool reversed) + { + var referenceSearchParameter = new SearchParameterInfo( + name: "subject", + code: "subject", + searchParamType: SearchParamType.Reference, + url: new Uri("http://hl7.org/fhir/SearchParameter/Observation-subject"), + targetResourceTypes: new[] { KnownResourceTypes.Patient }); + var vectorSearchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/observation-semantic")); + var expression = new ChainedExpression( + new[] { KnownResourceTypes.Observation }, + referenceSearchParameter, + new[] { KnownResourceTypes.Patient }, + reversed, + new VectorSearchExpression(vectorSearchParameter, "breathing difficulty")); + + Assert.Null(expression.AcceptVisitor(RemoveVectorSearchRewriter.Instance)); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/SqlServerSortingValidatorTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/SqlServerSortingValidatorTests.cs index 9cabbd508a..84c80d0ded 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/SqlServerSortingValidatorTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/SqlServerSortingValidatorTests.cs @@ -49,6 +49,18 @@ public void GivenSupportedSortParametersType_WhenValidating_ThenReturnsTrue(Sear Assert.Empty(errorMessage); } + [Fact] + public void GivenScoreSortParameter_WhenValidating_ThenReturnsTrue() + { + IReadOnlyList<(SearchParameterInfo, SortOrder)> searchList = + [(SearchParameterInfo.ScoreSearchParameter, SortOrder.Ascending)]; + + bool sortingValid = _sqlServerSortingValidator.ValidateSorting(searchList, out IReadOnlyList errorMessages); + + Assert.True(sortingValid); + Assert.Empty(errorMessages); + } + [Theory] [MemberData(nameof(GetSupportedSearchParamTypes))] public void GivenSupportedSortParametersTypeForSchemaOlderThanV17_WhenValidating_ThenReturnsFalse(SearchParamType searchParamType) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/TopRewriterTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/TopRewriterTests.cs index 706d7bb283..16d2d09d73 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/TopRewriterTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/Visitors/TopRewriterTests.cs @@ -4,9 +4,13 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Linq; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.SqlServer.Features.Search; using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions; using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors; using Microsoft.Health.Fhir.Tests.Common; @@ -95,6 +99,42 @@ public void GivenNormalQuery_WhenVisited_ThenAddsTopExpression() Assert.Null(result.SearchParamTableExpressions[1].Predicate); } + [Fact] + public void GivenPreparedVectorQuery_WhenVisited_ThenDoesNotLimitStructuredCandidates() + { + // Arrange + var searchParamExpression = new SearchParameterExpression( + TestSearchParam, + Expression.StringEquals(FieldName.String, null, "test", ignoreCase: false)); + var sqlRoot = SqlRootExpression.WithSearchParamTableExpressions( + new SearchParamTableExpression(null, searchParamExpression, SearchParamTableExpressionKind.Normal)); + var vectorSearchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/semantic-text")); + var searchOptions = new SqlSearchOptions(new SearchOptions + { + CountOnly = false, + SearchParameters = Array.Empty(), + UnsupportedSearchParams = Array.Empty>(), + Sort = Array.Empty<(SearchParameterInfo, SortOrder)>(), + }) + { + PreparedVectorQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 1, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray()), + }; + + // Act + Expression result = TopRewriter.Instance.VisitSqlRoot(sqlRoot, searchOptions); + + // Assert + Assert.Same(sqlRoot, result); + Assert.Single(((SqlRootExpression)result).SearchParamTableExpressions); + } + [Fact] public void GivenQueryWithMultipleExpressions_WhenVisited_ThenAddsTopExpressionAtEnd() { diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SemanticSearch/SqlVectorStoreTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SemanticSearch/SqlVectorStoreTests.cs new file mode 100644 index 0000000000..7c09a19d9e --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SemanticSearch/SqlVectorStoreTests.cs @@ -0,0 +1,213 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.SqlServer.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.SemanticSearch +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public class SqlVectorStoreTests + { + private const short TestResourceTypeId = 100; + private const short TestSearchParamId = 1; + private const short TestEmbeddingModelId = 1; + private const short ReplacementEmbeddingModelId = 2; + + // Opt-in integration test: it only runs when the database is configured through an environment variable, + // so CI (which has neither credentials nor network access to the database) stays offline. + // To run locally: az login, then set FHIR_TEST_SQL_CONNECTIONSTRING to the FHIR database connection string + // (for example "Server=tcp:.database.windows.net,1433;Initial Catalog=;Authentication=Active Directory Default;Encrypt=True;"). + [Fact] + public async Task GivenChunks_WhenStored_ThenTheyArePersistedToVectorSearchParam() + { + string connectionString = Environment.GetEnvironmentVariable("FHIR_TEST_SQL_CONNECTIONSTRING"); + + if (string.IsNullOrWhiteSpace(connectionString)) + { + return; + } + + long resourceSurrogateId = DateTime.UtcNow.Ticks; + + float[] embedding = await CreateEmbeddingAsync(); + var chunks = new List + { + new VectorSearchChunk(0, "first passage", new byte[32], embedding), + new VectorSearchChunk(1, "second passage", new byte[32], embedding), + }; + + var store = new SqlVectorStore(connectionString); + + try + { + await store.StoreAsync(TestResourceTypeId, resourceSurrogateId, TestSearchParamId, TestEmbeddingModelId, chunks, CancellationToken.None); + + int count = await CountAsync(connectionString, resourceSurrogateId); + Assert.Equal(2, count); + + await store.StoreAsync( + TestResourceTypeId, + resourceSurrogateId, + TestSearchParamId, + ReplacementEmbeddingModelId, + new[] { chunks[0] }, + CancellationToken.None); + + count = await CountAsync(connectionString, resourceSurrogateId); + Assert.Equal(1, count); + + await store.StoreAsync( + TestResourceTypeId, + resourceSurrogateId, + TestSearchParamId, + ReplacementEmbeddingModelId, + Array.Empty(), + CancellationToken.None); + + count = await CountAsync(connectionString, resourceSurrogateId); + Assert.Equal(0, count); + } + finally + { + await DeleteAsync(connectionString, resourceSurrogateId); + } + } + + [Fact] + public async Task GivenStoredVectors_WhenSearching_ThenTheClosestPassageRanksFirst() + { + string connectionString = Environment.GetEnvironmentVariable("FHIR_TEST_SQL_CONNECTIONSTRING"); + + if (string.IsNullOrWhiteSpace(connectionString)) + { + return; + } + + long surrogateChestPain = DateTime.UtcNow.Ticks; + long surrogateFracture = surrogateChestPain + 1; + + var client = new DeterministicEmbeddingClient(dimensions: 1536); + IReadOnlyList vectors = await client.GenerateEmbeddingsAsync(new[] { "chest pain", "fractured femur" }, CancellationToken.None); + float[] chestPain = vectors[0]; + float[] fracture = vectors[1]; + + var store = new SqlVectorStore(connectionString); + + try + { + await store.StoreAsync( + TestResourceTypeId, + surrogateChestPain, + TestSearchParamId, + TestEmbeddingModelId, + new[] + { + new VectorSearchChunk(0, "chest pain", new byte[32], chestPain), + new VectorSearchChunk(1, "fractured femur", new byte[32], fracture), + }, + CancellationToken.None); + await store.StoreAsync(TestResourceTypeId, surrogateFracture, TestSearchParamId, TestEmbeddingModelId, new[] { new VectorSearchChunk(0, "fractured femur", new byte[32], fracture) }, CancellationToken.None); + await SetSourceProvenanceAsync(connectionString, surrogateChestPain, chunkOrdinal: 0); + + IReadOnlyList results = await store.SearchAsync( + TestResourceTypeId, + TestSearchParamId, + TestEmbeddingModelId, + VectorSearchConfiguration.SupportedDistanceMetric, + chestPain, + new[] { surrogateChestPain, surrogateFracture }, + maxResults: 2, + evidenceCount: 2, + CancellationToken.None); + + Assert.Equal(3, results.Count); + Assert.Equal(surrogateChestPain, results[0].ResourceSurrogateId); + Assert.Equal("chest pain", results[0].ChunkText); + Assert.Equal(0, results[0].ChunkOrdinal); + Assert.Equal(TestResourceTypeId, results[0].SourceResourceTypeId); + Assert.Equal("binary-1", results[0].SourceResourceId); + Assert.Equal("2", results[0].SourceResourceVersion); + Assert.Equal("Binary.data", results[0].SourcePath); + Assert.Equal(surrogateChestPain, results[1].ResourceSurrogateId); + Assert.Equal("fractured femur", results[1].ChunkText); + Assert.Equal(2, results.Select(result => result.ResourceSurrogateId).Distinct().Count()); + Assert.True(results[0].Score >= results[2].Score); + } + finally + { + await DeleteAsync(connectionString, surrogateChestPain); + await DeleteAsync(connectionString, surrogateFracture); + } + } + + private static async Task CreateEmbeddingAsync() + { + var client = new DeterministicEmbeddingClient(dimensions: 1536); + IReadOnlyList vectors = await client.GenerateEmbeddingsAsync(new[] { "chest pain" }, CancellationToken.None); + return vectors[0]; + } + + private static async Task CountAsync(string connectionString, long resourceSurrogateId) + { + await using var connection = new SqlConnection(connectionString); + await connection.OpenAsync(); + + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT COUNT(*) FROM dbo.VectorSearchParam WHERE ResourceTypeId = @rt AND ResourceSurrogateId = @rid;"; + command.Parameters.AddWithValue("@rt", TestResourceTypeId); + command.Parameters.AddWithValue("@rid", resourceSurrogateId); + + return (int)await command.ExecuteScalarAsync(); + } + + private static async Task SetSourceProvenanceAsync(string connectionString, long resourceSurrogateId, short chunkOrdinal) + { + await using var connection = new SqlConnection(connectionString); + await connection.OpenAsync(); + + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = @" +UPDATE dbo.VectorSearchParam +SET SourceResourceTypeId = @sourceResourceTypeId, + SourceResourceId = 'binary-1', + SourceResourceVersion = '2', + SourcePath = 'Binary.data' +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId + AND ChunkOrdinal = @chunkOrdinal;"; + command.Parameters.AddWithValue("@sourceResourceTypeId", TestResourceTypeId); + command.Parameters.AddWithValue("@resourceTypeId", TestResourceTypeId); + command.Parameters.AddWithValue("@resourceSurrogateId", resourceSurrogateId); + command.Parameters.AddWithValue("@chunkOrdinal", chunkOrdinal); + + await command.ExecuteNonQueryAsync(); + } + + private static async Task DeleteAsync(string connectionString, long resourceSurrogateId) + { + await using var connection = new SqlConnection(connectionString); + await connection.OpenAsync(); + + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = "DELETE FROM dbo.VectorSearchParam WHERE ResourceTypeId = @rt AND ResourceSurrogateId = @rid;"; + command.Parameters.AddWithValue("@rt", TestResourceTypeId); + command.Parameters.AddWithValue("@rid", resourceSurrogateId); + + await command.ExecuteNonQueryAsync(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlQueryGeneratorTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlQueryGeneratorTests.cs index 81a9e9206c..1ab784ba95 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlQueryGeneratorTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlQueryGeneratorTests.cs @@ -14,6 +14,7 @@ using Microsoft.Health.Fhir.Core.Features.Definition; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.SqlServer; using Microsoft.Health.Fhir.SqlServer.Features.Schema; @@ -152,6 +153,519 @@ public void GivenASearchTypeForHistorySoftDeleted_WhenSqlGenerated_ThenSqlFilter Assert.Contains("IsDeleted = 1", _strBuilder.ToString()); } + [Fact] + public void GivenPreparedVectorQueryAndStructuredCandidates_WhenSqlGenerated_ThenRanksBeforePagination() + { + // Arrange + var vectorSearchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/semantic-text")); + _fhirModel.GetSearchParamId(vectorSearchParameter.Url).Returns((short)71); + + Expression predicate = Expression.And( + [new SearchParameterExpression( + new SearchParameterInfo("_type", "_type"), + new StringExpression(StringOperator.Equals, FieldName.String, null, "Patient", false))]); + var sqlExpression = new SqlRootExpression( + [new SearchParamTableExpression(null, predicate, SearchParamTableExpressionKind.All)], + new List()); + var searchOptions = new SqlSearchOptions(new SearchOptions + { + MaxItemCount = 10, + SearchParameters = Array.Empty(), + UnsupportedSearchParams = Array.Empty>(), + Sort = Array.Empty<(SearchParameterInfo, SortOrder)>(), + ResourceVersionTypes = ResourceVersionType.Latest, + }) + { + PreparedVectorQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 3, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray(), + minimumScore: 0.65m), + }; + + // Act + _queryGenerator.VisitSqlRoot(sqlExpression, searchOptions); + string generatedSql = _strBuilder.ToString(); + + // Assert + int candidateJoinIndex = generatedSql.IndexOf("JOIN cte0", StringComparison.Ordinal); + int vectorApplyIndex = generatedSql.IndexOf("CROSS APPLY", StringComparison.Ordinal); + + Assert.True(candidateJoinIndex >= 0, generatedSql); + Assert.True(vectorApplyIndex > candidateJoinIndex, generatedSql); + Assert.Contains("SELECT TOP (", generatedSql, StringComparison.Ordinal); + Assert.Contains("dbo.VectorSearchParam", generatedSql, StringComparison.Ordinal); + Assert.Contains("VECTOR_DISTANCE(", generatedSql, StringComparison.Ordinal); + Assert.Contains("AS VECTOR(1536)", generatedSql, StringComparison.Ordinal); + Assert.Contains("semantic.SemanticDistance", generatedSql, StringComparison.Ordinal); + Assert.Contains("semantic.SemanticChunkOrdinal", generatedSql, StringComparison.Ordinal); + Assert.Contains("semantic.SemanticChunkText", generatedSql, StringComparison.Ordinal); + Assert.Contains("AS SemanticChunkOrdinal", generatedSql, StringComparison.Ordinal); + Assert.Contains("AS SemanticChunkText", generatedSql, StringComparison.Ordinal); + Assert.Contains("AS SemanticSourceResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("AS SemanticSourceResourceId", generatedSql, StringComparison.Ordinal); + Assert.Contains("AS SemanticSourceResourceVersion", generatedSql, StringComparison.Ordinal); + Assert.Contains("AS SemanticSourcePath", generatedSql, StringComparison.Ordinal); + Assert.Contains("AS SemanticEvidenceJson", generatedSql, StringComparison.Ordinal); + Assert.Contains("FOR JSON PATH", generatedSql, StringComparison.Ordinal); + Assert.Contains("AND v.SearchParamId =", generatedSql, StringComparison.Ordinal); + Assert.Contains("))) <= ", generatedSql, StringComparison.Ordinal); + Assert.Contains("ORDER BY SemanticDistance ASC", generatedSql, StringComparison.Ordinal); + Assert.DoesNotContain("[0.25,0.25", generatedSql, StringComparison.Ordinal); + + // The Top CTE that normally carries IsMatch/IsPartial is suppressed for vector search, + // so the outer projection must emit constant match bits instead of reading them from the last CTE. + Assert.Contains("CAST(1 AS bit) AS IsMatch", generatedSql, StringComparison.Ordinal); + Assert.Contains("CAST(0 AS bit) AS IsPartial", generatedSql, StringComparison.Ordinal); + Assert.DoesNotContain("CAST(IsMatch AS bit)", generatedSql, StringComparison.Ordinal); + Assert.DoesNotContain("CAST(IsPartial AS bit)", generatedSql, StringComparison.Ordinal); + } + + [Fact] + public void GivenPreparedReverseChainedVectorQuery_WhenSqlGenerated_ThenRanksRootsByRelatedWitnessVectors() + { + // Arrange + var vectorSearchParameter = new SearchParameterInfo( + name: "ObservationSemantic", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/observation-semantic"), + vectorConfig: new VectorSearchParameterConfig()); + var referenceSearchParameter = new SearchParameterInfo( + name: "subject", + code: "subject", + searchParamType: SearchParamType.Reference, + url: new Uri("http://hl7.org/fhir/SearchParameter/Observation-subject"), + targetResourceTypes: new[] { KnownResourceTypes.Patient }); + _fhirModel.GetSearchParamId(vectorSearchParameter.Url).Returns((short)71); + _fhirModel.GetSearchParamId(referenceSearchParameter.Url).Returns((short)72); + _fhirModel.GetResourceTypeId(KnownResourceTypes.Observation).Returns((short)103); + _fhirModel.GetResourceTypeId(KnownResourceTypes.Patient).Returns((short)104); + + Expression predicate = Expression.And( + [new SearchParameterExpression( + new SearchParameterInfo("_type", "_type"), + new StringExpression(StringOperator.Equals, FieldName.String, null, KnownResourceTypes.Patient, false))]); + var sqlExpression = new SqlRootExpression( + [new SearchParamTableExpression(null, predicate, SearchParamTableExpressionKind.All)], + new List()); + var searchOptions = new SqlSearchOptions(new SearchOptions + { + MaxItemCount = 10, + SearchParameters = Array.Empty(), + UnsupportedSearchParams = Array.Empty>(), + Sort = Array.Empty<(SearchParameterInfo, SortOrder)>(), + ResourceVersionTypes = ResourceVersionType.Latest, + }) + { + PreparedVectorQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 3, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray(), + minimumScore: 0.65m, + chainLinks: + [ + new PreparedVectorSearchChainLink( + new[] { KnownResourceTypes.Observation }, + referenceSearchParameter, + new[] { KnownResourceTypes.Patient }, + reversed: true), + ]), + }; + searchOptions.SemanticContinuationDistance = 0.125; + searchOptions.SemanticContinuationResourceTypeId = 104; + searchOptions.SemanticContinuationResourceSurrogateId = 12345; + + // Act + _queryGenerator.VisitSqlRoot(sqlExpression, searchOptions); + string generatedSql = _strBuilder.ToString(); + + // Assert + Assert.Contains("FROM dbo.ReferenceSearchParam AS semanticReference", generatedSql, StringComparison.Ordinal); + Assert.Contains("JOIN dbo.Resource AS semanticWitness", generatedSql, StringComparison.Ordinal); + Assert.Contains("JOIN dbo.VectorSearchParam AS v", generatedSql, StringComparison.Ordinal); + Assert.Contains("SELECT DISTINCT r.ResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("SELECT TOP (1) VECTOR_DISTANCE", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticReference.ReferenceResourceTypeId = r.ResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticReference.ReferenceResourceId = r.ResourceId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.ResourceTypeId IN", generatedSql, StringComparison.Ordinal); + _fhirModel.Received(2).GetResourceTypeId(KnownResourceTypes.Observation); + Assert.Contains("semanticWitness.IsHistory = 0", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.IsDeleted = 0", generatedSql, StringComparison.Ordinal); + Assert.Contains("ev.ResourceTypeId = v.ResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("ev.ResourceSurrogateId = v.ResourceSurrogateId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.ResourceTypeId AS witnessResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.ResourceId AS witnessResourceId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.Version AS witnessResourceVersion", generatedSql, StringComparison.Ordinal); + Assert.Contains("semantic.SemanticDistance >", generatedSql, StringComparison.Ordinal); + Assert.Contains("r.ResourceSurrogateId >", generatedSql, StringComparison.Ordinal); + Assert.Contains("ORDER BY SemanticDistance ASC", generatedSql, StringComparison.Ordinal); + } + + [Fact] + public void GivenPreparedForwardChainedVectorQuery_WhenSqlGenerated_ThenRanksRootsByReferencedTargetVectors() + { + // Arrange + var vectorSearchParameter = new SearchParameterInfo( + name: "PatientSemantic", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/patient-semantic"), + vectorConfig: new VectorSearchParameterConfig()); + var referenceSearchParameter = new SearchParameterInfo( + name: "subject", + code: "subject", + searchParamType: SearchParamType.Reference, + url: new Uri("http://hl7.org/fhir/SearchParameter/Observation-subject"), + targetResourceTypes: new[] { KnownResourceTypes.Patient }); + _fhirModel.GetSearchParamId(vectorSearchParameter.Url).Returns((short)71); + _fhirModel.GetSearchParamId(referenceSearchParameter.Url).Returns((short)72); + _fhirModel.GetResourceTypeId(KnownResourceTypes.Observation).Returns((short)103); + _fhirModel.GetResourceTypeId(KnownResourceTypes.Patient).Returns((short)104); + + Expression predicate = Expression.And( + [new SearchParameterExpression( + new SearchParameterInfo("_type", "_type"), + new StringExpression(StringOperator.Equals, FieldName.String, null, KnownResourceTypes.Observation, false))]); + var sqlExpression = new SqlRootExpression( + [new SearchParamTableExpression(null, predicate, SearchParamTableExpressionKind.All)], + new List()); + var searchOptions = new SqlSearchOptions(new SearchOptions + { + MaxItemCount = 10, + SearchParameters = Array.Empty(), + UnsupportedSearchParams = Array.Empty>(), + Sort = Array.Empty<(SearchParameterInfo, SortOrder)>(), + ResourceVersionTypes = ResourceVersionType.Latest, + }) + { + PreparedVectorQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 3, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray(), + minimumScore: 0.65m, + chainLinks: + [ + new PreparedVectorSearchChainLink( + new[] { KnownResourceTypes.Observation }, + referenceSearchParameter, + new[] { KnownResourceTypes.Patient }, + reversed: false), + ]), + }; + searchOptions.SemanticContinuationDistance = 0.125; + searchOptions.SemanticContinuationResourceTypeId = 103; + searchOptions.SemanticContinuationResourceSurrogateId = 12345; + + // Act + _queryGenerator.VisitSqlRoot(sqlExpression, searchOptions); + string generatedSql = _strBuilder.ToString(); + + // Assert + Assert.Contains("FROM dbo.ReferenceSearchParam AS semanticReference", generatedSql, StringComparison.Ordinal); + Assert.Contains("JOIN dbo.Resource AS semanticWitness", generatedSql, StringComparison.Ordinal); + Assert.Contains("SELECT DISTINCT r.ResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("SELECT TOP (1) VECTOR_DISTANCE", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.ResourceTypeId = semanticReference.ReferenceResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.ResourceId = semanticReference.ReferenceResourceId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticReference.ResourceTypeId = r.ResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticReference.ResourceSurrogateId = r.ResourceSurrogateId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.IsHistory = 0", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.IsDeleted = 0", generatedSql, StringComparison.Ordinal); + Assert.Contains("ev.ResourceTypeId = v.ResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("ev.ResourceSurrogateId = v.ResourceSurrogateId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.ResourceTypeId AS witnessResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.ResourceId AS witnessResourceId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.Version AS witnessResourceVersion", generatedSql, StringComparison.Ordinal); + Assert.Contains("semantic.SemanticDistance >", generatedSql, StringComparison.Ordinal); + Assert.Contains("r.ResourceSurrogateId >", generatedSql, StringComparison.Ordinal); + Assert.Contains("v.ResourceTypeId ASC, v.ResourceSurrogateId ASC, v.ChunkOrdinal ASC", generatedSql, StringComparison.Ordinal); + Assert.Contains("ORDER BY SemanticDistance ASC", generatedSql, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void GivenChainedLinkedSourceVectorQuery_WhenSqlGenerated_ThenWitnessAndSourceProvenanceAreProjected(bool reversed) + { + var vectorSearchParameter = new SearchParameterInfo( + name: "DocumentReferenceSemantic", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/document-reference-semantic"), + vectorConfig: new VectorSearchParameterConfig { SourceStrategy = VectorTextSourceStrategy.LocalBinaryReference }); + var referenceSearchParameter = new SearchParameterInfo( + name: "subject", + code: "subject", + searchParamType: SearchParamType.Reference, + url: new Uri("http://hl7.org/fhir/SearchParameter/DocumentReference-subject"), + targetResourceTypes: new[] { KnownResourceTypes.Patient }); + _fhirModel.GetSearchParamId(vectorSearchParameter.Url).Returns((short)71); + _fhirModel.GetSearchParamId(referenceSearchParameter.Url).Returns((short)72); + _fhirModel.GetResourceTypeId(KnownResourceTypes.DocumentReference).Returns((short)103); + _fhirModel.GetResourceTypeId(KnownResourceTypes.Patient).Returns((short)104); + + var sqlExpression = new SqlRootExpression( + resourceTableExpressions: new List(), + searchParamTableExpressions: new List()); + var searchOptions = new SqlSearchOptions(new SearchOptions + { + MaxItemCount = 10, + SearchParameters = Array.Empty(), + UnsupportedSearchParams = Array.Empty>(), + Sort = Array.Empty<(SearchParameterInfo, SortOrder)>(), + ResourceVersionTypes = ResourceVersionType.Latest, + }) + { + PreparedVectorQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 3, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray(), + chainLinks: + [ + new PreparedVectorSearchChainLink( + new[] { KnownResourceTypes.DocumentReference }, + referenceSearchParameter, + new[] { KnownResourceTypes.Patient }, + reversed), + ]), + }; + + _queryGenerator.VisitSqlRoot(sqlExpression, searchOptions); + string generatedSql = _strBuilder.ToString(); + + Assert.Contains("v.SourceResourceTypeId AS SemanticSourceResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("ev.SourceResourceId AS sourceResourceId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.ResourceTypeId AS witnessResourceTypeId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.ResourceId AS witnessResourceId", generatedSql, StringComparison.Ordinal); + Assert.Contains("semanticWitness.Version AS witnessResourceVersion", generatedSql, StringComparison.Ordinal); + } + + [Fact] + public void GivenMultiHopVectorQuery_WhenSqlGenerated_ThenSearchIsRejected() + { + var vectorSearchParameter = new SearchParameterInfo( + name: "PatientSemantic", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/patient-semantic"), + vectorConfig: new VectorSearchParameterConfig()); + var subjectSearchParameter = new SearchParameterInfo( + name: "subject", + code: "subject", + searchParamType: SearchParamType.Reference, + url: new Uri("http://hl7.org/fhir/SearchParameter/Observation-subject"), + targetResourceTypes: new[] { KnownResourceTypes.Patient }); + var generalPractitionerSearchParameter = new SearchParameterInfo( + name: "general-practitioner", + code: "general-practitioner", + searchParamType: SearchParamType.Reference, + url: new Uri("http://hl7.org/fhir/SearchParameter/Patient-general-practitioner"), + targetResourceTypes: new[] { KnownResourceTypes.Practitioner }); + _fhirModel.GetSearchParamId(vectorSearchParameter.Url).Returns((short)71); + + var sqlExpression = new SqlRootExpression( + resourceTableExpressions: new List(), + searchParamTableExpressions: new List()); + var searchOptions = new SqlSearchOptions(new SearchOptions + { + MaxItemCount = 10, + SearchParameters = Array.Empty(), + UnsupportedSearchParams = Array.Empty>(), + Sort = Array.Empty<(SearchParameterInfo, SortOrder)>(), + ResourceVersionTypes = ResourceVersionType.Latest, + }) + { + PreparedVectorQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 3, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray(), + chainLinks: + [ + new PreparedVectorSearchChainLink( + new[] { KnownResourceTypes.Observation }, + subjectSearchParameter, + new[] { KnownResourceTypes.Patient }, + reversed: false), + new PreparedVectorSearchChainLink( + new[] { KnownResourceTypes.Patient }, + generalPractitionerSearchParameter, + new[] { KnownResourceTypes.Practitioner }, + reversed: false), + ]), + }; + + Assert.Throws(() => _queryGenerator.VisitSqlRoot(sqlExpression, searchOptions)); + } + + [Fact] + public void GivenPreparedVectorQueryAndScoreSort_WhenSqlGenerated_ThenRanksByDistanceWithoutSortValueLookup() + { + // Arrange + var vectorSearchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/semantic-text")); + _fhirModel.GetSearchParamId(vectorSearchParameter.Url).Returns((short)71); + Expression predicate = Expression.And( + [new SearchParameterExpression( + new SearchParameterInfo("_type", "_type"), + new StringExpression(StringOperator.Equals, FieldName.String, null, "Patient", false))]); + var sqlExpression = new SqlRootExpression( + [new SearchParamTableExpression(null, predicate, SearchParamTableExpressionKind.All)], + new List()); + var searchOptions = CreateVectorSearchOptions( + vectorSearchParameter, + [ + (SearchParameterInfo.ScoreSearchParameter, SortOrder.Ascending), + (SearchParameterInfo.ResourceTypeSearchParameter, SortOrder.Ascending), + (new SearchParameterInfo(SearchParameterNames.LastUpdated, SearchParameterNames.LastUpdated), SortOrder.Ascending), + ]); + + // Act + Expression rewritten = new SortRewriter(_queryGeneratorFactory).VisitSqlRoot(sqlExpression, searchOptions); + _queryGenerator.VisitSqlRoot((SqlRootExpression)rewritten, searchOptions); + string generatedSql = _strBuilder.ToString(); + + // Assert + Assert.Same(sqlExpression, rewritten); + Assert.Contains("ORDER BY SemanticDistance ASC", generatedSql, StringComparison.Ordinal); + Assert.DoesNotContain("SortValue", generatedSql, StringComparison.Ordinal); + } + + [Fact] + public void GivenPreparedVectorQueryAndSemanticCursor_WhenSqlGenerated_ThenContinuesAfterDistanceAndStableKeys() + { + // Arrange + var vectorSearchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/semantic-text")); + _fhirModel.GetSearchParamId(vectorSearchParameter.Url).Returns((short)71); + Expression predicate = Expression.And( + [new SearchParameterExpression( + new SearchParameterInfo("_type", "_type"), + new StringExpression(StringOperator.Equals, FieldName.String, null, "Patient", false))]); + var sqlExpression = new SqlRootExpression( + [new SearchParamTableExpression(null, predicate, SearchParamTableExpressionKind.All)], + new List()); + var searchOptions = CreateVectorSearchOptions( + vectorSearchParameter, + [(SearchParameterInfo.ScoreSearchParameter, SortOrder.Ascending)]); + searchOptions.SemanticContinuationDistance = 0.125; + searchOptions.SemanticContinuationResourceTypeId = 103; + searchOptions.SemanticContinuationResourceSurrogateId = 12345; + + // Act + _queryGenerator.VisitSqlRoot(sqlExpression, searchOptions); + string generatedSql = _strBuilder.ToString(); + + // Assert + Assert.Contains("semantic.SemanticDistance >", generatedSql, StringComparison.Ordinal); + Assert.Contains("semantic.SemanticDistance =", generatedSql, StringComparison.Ordinal); + Assert.Contains("ResourceTypeId >", generatedSql, StringComparison.Ordinal); + Assert.Contains("ResourceSurrogateId >", generatedSql, StringComparison.Ordinal); + Assert.Contains("ORDER BY SemanticDistance ASC", generatedSql, StringComparison.Ordinal); + } + + [Fact] + public void GivenPreparedVectorQueryAndLastUpdatedSort_WhenSqlGenerated_ThenRequestedSortOverridesRelevanceOrder() + { + // Arrange + var vectorSearchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/semantic-text")); + _fhirModel.GetSearchParamId(vectorSearchParameter.Url).Returns((short)71); + + Expression predicate = Expression.And( + [new SearchParameterExpression( + new SearchParameterInfo("_type", "_type"), + new StringExpression(StringOperator.Equals, FieldName.String, null, "Patient", false))]); + var sqlExpression = new SqlRootExpression( + [new SearchParamTableExpression(null, predicate, SearchParamTableExpressionKind.All)], + new List()); + var searchOptions = CreateVectorSearchOptions( + vectorSearchParameter, + [(new SearchParameterInfo(SearchParameterNames.LastUpdated, SearchParameterNames.LastUpdated), SortOrder.Descending)]); + + // Act + _queryGenerator.VisitSqlRoot(sqlExpression, searchOptions); + string generatedSql = _strBuilder.ToString(); + + // Assert + Assert.Contains("ResourceSurrogateId DESC", generatedSql, StringComparison.Ordinal); + Assert.DoesNotContain("ORDER BY SemanticDistance ASC", generatedSql, StringComparison.Ordinal); + Assert.Contains("semantic.SemanticDistance", generatedSql, StringComparison.Ordinal); + Assert.Contains("semantic.SemanticEvidenceJson", generatedSql, StringComparison.Ordinal); + } + + [Fact] + public void GivenPreparedVectorQueryAndDateSort_WhenSqlGenerated_ThenRequestedSortOverridesRelevanceOrder() + { + // Arrange + var vectorSearchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/semantic-text")); + var dateSortParameter = new SearchParameterInfo( + name: "date", + code: "date", + searchParamType: SearchParamType.Date, + url: new Uri("https://example.org/fhir/SearchParameter/date")); + _fhirModel.GetSearchParamId(vectorSearchParameter.Url).Returns((short)71); + _fhirModel.GetSearchParamId(dateSortParameter.Url).Returns((short)72); + + Expression predicate = Expression.And( + [new SearchParameterExpression( + new SearchParameterInfo("_type", "_type"), + new StringExpression(StringOperator.Equals, FieldName.String, null, "Patient", false))]); + var sqlExpression = new SqlRootExpression( + [new SearchParamTableExpression(null, predicate, SearchParamTableExpressionKind.All)], + new List()); + var searchOptions = CreateVectorSearchOptions(vectorSearchParameter, [(dateSortParameter, SortOrder.Descending)]); + sqlExpression = (SqlRootExpression)new SortRewriter(_queryGeneratorFactory).VisitSqlRoot(sqlExpression, searchOptions); + + // Act + _queryGenerator.VisitSqlRoot(sqlExpression, searchOptions); + string generatedSql = _strBuilder.ToString(); + + // Assert + Assert.Contains("SortValue DESC", generatedSql, StringComparison.Ordinal); + Assert.DoesNotContain("ORDER BY SemanticDistance ASC", generatedSql, StringComparison.Ordinal); + Assert.Contains("semantic.SemanticDistance", generatedSql, StringComparison.Ordinal); + Assert.Contains("semantic.SemanticEvidenceJson", generatedSql, StringComparison.Ordinal); + } + + private static SqlSearchOptions CreateVectorSearchOptions( + SearchParameterInfo vectorSearchParameter, + IReadOnlyList<(SearchParameterInfo searchParameterInfo, SortOrder sortOrder)> sort) + { + return new SqlSearchOptions(new SearchOptions + { + MaxItemCount = 10, + SearchParameters = Array.Empty(), + UnsupportedSearchParams = Array.Empty>(), + Sort = sort, + ResourceVersionTypes = ResourceVersionType.Latest, + }) + { + PreparedVectorQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 3, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray(), + minimumScore: 0.65m), + }; + } + [Fact] public void GivenReferenceSearchParameterWithMultipleTargetTypes_WhenSqlGenerated_ThenSqlIncludesOrClauseForReferenceResourceTypeId() { diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceTests.cs index 8464541d15..ae67989989 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceTests.cs @@ -21,6 +21,7 @@ using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.SqlServer.Features.Schema; using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; @@ -59,10 +60,15 @@ public class SqlServerSearchServiceTests private readonly RequestContextAccessor _requestContextAccessor; private readonly ISqlQueryHashCalculator _queryHashCalculator; private readonly IQueryPlanReuseChecker _queryPlanReuseChecker; + private readonly IVectorSearchQueryProcessor _vectorSearchQueryProcessor; private readonly SqlServerSearchService _searchService; public SqlServerSearchServiceTests() { + ModelInfoProvider.SetProvider( + MockModelInfoProviderBuilder.Create(FhirSpecification.R4) + .AddKnownTypes(KnownResourceTypes.DocumentReference) + .Build()); _searchOptionsFactory = Substitute.For(); _fhirDataStore = Substitute.For(); _model = Substitute.For(); @@ -72,6 +78,7 @@ public SqlServerSearchServiceTests() _requestContextAccessor = Substitute.For>(); _queryHashCalculator = Substitute.For(); _queryPlanReuseChecker = Substitute.For(); + _vectorSearchQueryProcessor = Substitute.For(); var config = new SqlServerDataStoreConfiguration { @@ -116,7 +123,8 @@ public SqlServerSearchServiceTests() _compressedRawResourceConverter, _queryHashCalculator, _queryPlanReuseChecker, - NullLogger.Instance); + NullLogger.Instance, + _vectorSearchQueryProcessor); } [Fact] @@ -305,6 +313,175 @@ public void Model_Property_ReturnsInjectedModel() Assert.Same(_model, model); } + [Fact] + public async Task GivenSemanticSearchWithExplicitSort_WhenSearching_ThenVectorQueryIsPrepared() + { + // Arrange + var vectorSearchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/semantic-text")); + var expectedException = new InvalidOperationException("Stop after vector query preparation."); + _vectorSearchQueryProcessor + .PrepareAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromException(expectedException)); + var searchOptions = new SearchOptions + { + MaxItemCount = 10, + Expression = new VectorSearchExpression(vectorSearchParameter, "breathing difficulty"), + SearchParameters = Array.Empty(), + UnsupportedSearchParams = Array.Empty>(), + Sort = new[] { (new SearchParameterInfo(SearchParameterNames.LastUpdated, SearchParameterNames.LastUpdated), SortOrder.Descending) }, + }; + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync( + () => _searchService.SearchAsync(searchOptions, CancellationToken.None)); + + // Assert + Assert.Same(expectedException, exception); + await _vectorSearchQueryProcessor.Received(1).PrepareAsync(searchOptions.Expression, Arg.Any()); + } + + [Fact] + public void GivenChainedLinkedSourceEvidenceJson_WhenDeserialized_ThenWitnessAndSourceAreDistinctVersionedReferences() + { + var vectorSearchParameter = new SearchParameterInfo( + name: "DocumentReferenceSemantic", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/document-reference-semantic"), + expression: "DocumentReference.content.attachment.url", + vectorConfig: new VectorSearchParameterConfig { SourceStrategy = VectorTextSourceStrategy.LocalBinaryReference }); + var preparedQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 3, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray()); + ResourceWrapper root = CreateResourceWrapper("Patient", "patient", "1", 100); + _model.GetResourceTypeName(22).Returns("Binary"); + _model.GetResourceTypeName(105).Returns("Observation"); + const string evidenceJson = """ + [{ + "chunkOrdinal": 0, + "text": "Matched Binary passage", + "distance": 0.2, + "sourceResourceTypeId": 22, + "sourceResourceId": "binary", + "sourceResourceVersion": "2", + "sourcePath": "Binary.data", + "witnessResourceTypeId": 105, + "witnessResourceId": "document", + "witnessResourceVersion": 3 + }] + """; + + SemanticSearchEvidence evidence = Assert.Single( + _searchService.DeserializeSemanticEvidence(evidenceJson, preparedQuery, root)); + + Assert.Equal("Binary/binary/_history/2", evidence.SourceReference); + Assert.Equal("Observation/document/_history/3", evidence.WitnessReference); + Assert.Equal("Binary.data", evidence.SourcePath); + } + + [Theory] + [InlineData("not-json")] + [InlineData("[{\"text\":\"passage\",\"witnessResourceVersion\":\"invalid\"}]")] + public void GivenMalformedSemanticEvidenceJson_WhenDeserialized_ThenEvidenceIsDiscarded(string evidenceJson) + { + var vectorSearchParameter = new SearchParameterInfo( + name: "DocumentReferenceSemantic", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/document-reference-semantic"), + expression: "DocumentReference.content.attachment.url", + vectorConfig: new VectorSearchParameterConfig { SourceStrategy = VectorTextSourceStrategy.LocalBinaryReference }); + var preparedQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 3, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray()); + ResourceWrapper root = CreateResourceWrapper("Patient", "patient", "1", 100); + + IReadOnlyList evidence = _searchService.DeserializeSemanticEvidence(evidenceJson, preparedQuery, root); + + Assert.Empty(evidence); + } + + [Theory] + [InlineData(true, TotalType.None)] + [InlineData(false, TotalType.Accurate)] + public async Task GivenLinkedSourceSemanticSearchWithExactTotal_WhenSearching_ThenSearchIsRejected( + bool countOnly, + TotalType includeTotal) + { + var vectorSearchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/semantic-text"), + vectorConfig: new VectorSearchParameterConfig + { + SourceStrategy = VectorTextSourceStrategy.LocalBinaryReference, + }); + var preparedQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 3, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray()); + _vectorSearchQueryProcessor + .PrepareAsync(Arg.Any(), Arg.Any()) + .Returns(preparedQuery); + var searchOptions = new SearchOptions + { + CountOnly = countOnly, + IncludeTotal = includeTotal, + MaxItemCount = 10, + Expression = new VectorSearchExpression(vectorSearchParameter, "breathing difficulty"), + SearchParameters = Array.Empty(), + UnsupportedSearchParams = Array.Empty>(), + Sort = Array.Empty<(SearchParameterInfo, SortOrder)>(), + }; + + InvalidSearchOperationException exception = await Assert.ThrowsAsync( + () => _searchService.SearchAsync(searchOptions, CancellationToken.None)); + + Assert.Contains("localBinaryReference", exception.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void GivenSemanticRelevanceSort_WhenSortUpdated_ThenDistanceAndStableKeysArePreserved(bool explicitScoreSort) + { + var vectorSearchParameter = new SearchParameterInfo( + name: "SemanticText", + code: "semantic-text", + searchParamType: SearchParamType.Special, + url: new Uri("https://example.org/fhir/SearchParameter/semantic-text")); + var searchOptions = new SqlSearchOptions(new SearchOptions + { + SearchParameters = Array.Empty(), + UnsupportedSearchParams = Array.Empty>(), + Sort = explicitScoreSort + ? [(SearchParameterInfo.ScoreSearchParameter, SortOrder.Ascending)] + : [], + ResourceVersionTypes = ResourceVersionType.Latest, + }) + { + PreparedVectorQuery = new PreparedVectorSearchQuery( + vectorSearchParameter, + embeddingModelId: 3, + Enumerable.Repeat(0.25f, VectorSearchConfiguration.SupportedDimensions).ToArray(), + minimumScore: 0.65m), + }; + + SqlSearchOptions updated = _searchService.UpdateSort(searchOptions, searchExpression: null); + + Assert.Equal( + [SearchParameterNames.Score, SearchParameterNames.ResourceType, SearchParameterNames.LastUpdated], + updated.Sort.Select(sort => sort.searchParameterInfo.Name)); + Assert.All(updated.Sort, sort => Assert.Equal(SortOrder.Ascending, sort.sortOrder)); + } + public static IEnumerable SingleColumnTableData() { yield return new object[] { VLatest.TokenSearchParam.TableName, VLatest.TokenSearchParam.Code.Metadata.Name }; @@ -312,6 +489,26 @@ public static IEnumerable SingleColumnTableData() yield return new object[] { VLatest.ReferenceSearchParam.TableName, VLatest.ReferenceSearchParam.ReferenceResourceId.Metadata.Name }; } + private static ResourceWrapper CreateResourceWrapper( + string resourceType, + string resourceId, + string version, + long resourceSurrogateId) + { + return new ResourceWrapper( + resourceId, + version, + resourceType, + new RawResource($"{{\"resourceType\":\"{resourceType}\",\"id\":\"{resourceId}\"}}", FhirResourceFormat.Json, isMetaSet: false), + request: null, + DateTimeOffset.MinValue, + deleted: false, + searchIndices: null, + compartmentIndices: null, + lastModifiedClaims: null, + resourceSurrogateId: resourceSurrogateId); + } + [Theory] [MemberData(nameof(SingleColumnTableData))] public void GetKeyColumns_ForSingleColumnTable_ReturnsOnlyExpectedColumn(string tableName, string expectedColumn) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlSesrverSearchParameterValidatorTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlSesrverSearchParameterValidatorTests.cs index 76b4c12a36..a93233cf78 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlSesrverSearchParameterValidatorTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlSesrverSearchParameterValidatorTests.cs @@ -49,6 +49,14 @@ public static IEnumerable GetValidSearchParameters() yield return new object[] { new SearchParameterInfo("test", "test", ValueSets.SearchParamType.Reference) }; yield return new object[] { new SearchParameterInfo("test", "test", ValueSets.SearchParamType.String) }; yield return new object[] { new SearchParameterInfo("test", "test", ValueSets.SearchParamType.Uri) }; + yield return new object[] + { + new SearchParameterInfo( + "test", + "test", + ValueSets.SearchParamType.Special, + vectorConfig: new VectorSearchParameterConfig()), + }; var components = new List(); var component = new SearchParameterComponentInfo(); @@ -64,6 +72,14 @@ public static IEnumerable GetValidSearchParameters() public static IEnumerable GetInValidSearchParameters() { yield return new object[] { new SearchParameterInfo("test", "test", ValueSets.SearchParamType.Special) }; + yield return new object[] + { + new SearchParameterInfo( + "test", + "test", + ValueSets.SearchParamType.String, + vectorConfig: new VectorSearchParameterConfig()), + }; var components = new List(); var component = new SearchParameterComponentInfo(); diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs index b96f069f66..173c782467 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Storage/SqlServerFhirDataStoreUnitTests.cs @@ -26,11 +26,16 @@ using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Features.Search.Registry; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Core.UnitTests.Extensions; using Microsoft.Health.Fhir.SqlServer.Features.Schema; +using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; using Microsoft.Health.Fhir.SqlServer.Features.Storage; +using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration; +using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration.Merge; using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.ValueSets; using Microsoft.Health.SqlServer; using Microsoft.Health.SqlServer.Configs; using Microsoft.Health.SqlServer.Features.Client; @@ -165,6 +170,64 @@ private static ResourceWrapper CreateResourceWrapper(string rawResourceData) null); } + [Fact] + public void GivenVectorSearchIndices_WhenGeneratingMergeRows_ThenAllVectorMetadataIsPreserved() + { + // Arrange + SqlServerFhirDataStore dataStore = CreateSqlServerFhirDataStore(Substitute.For()); + SqlServerFhirModel model = GetModel(dataStore); + var canonicalUri = new Uri("https://example.org/fhir/SearchParameter/patient-summary-vector"); + typeof(SqlServerFhirModel) + .GetField("_searchParamUriToId", BindingFlags.NonPublic | BindingFlags.Instance) + .SetValue(model, new Dictionary()); + model.TryAddSearchParamIdToUriMapping(canonicalUri.OriginalString, 17); + var searchParameter = new SearchParameterInfo( + "PatientSummaryVector", + "summary-vector", + SearchParamType.Special, + canonicalUri); + ResourceWrapper resource = CreateResourceWrapper("{}"); + resource.ResourceSurrogateId = 42; + resource.UpdateVectorSearchIndices( + new[] + { + new VectorSearchIndexEntry( + searchParameter, + embeddingModelId: 5, + new[] + { + new VectorSearchChunk( + 3, + "clinical passage", + new byte[32], + new[] { 0.25f, -1.5f }, + "Patient", + "source-id", + "9", + "Patient.note.text"), + }), + }); + var generator = new VectorSearchParamListRowGenerator(model); + + // Act + VectorSearchParamListRow row = Assert.Single( + generator.GenerateRows(new[] { new MergeResourceWrapper(resource, keepHistory: true, hasVersionToCompare: true) })); + + // Assert + Assert.Equal(1, row.ResourceTypeId); + Assert.Equal(42, row.ResourceSurrogateId); + Assert.Equal(17, row.SearchParamId); + Assert.Equal(3, row.ChunkOrdinal); + Assert.Equal(5, row.EmbeddingModelId); + Assert.Equal("clinical passage", row.ChunkText); + Assert.Equal(new byte[32], row.SourceTextHash); + Assert.Equal(1, row.SourceResourceTypeId); + Assert.Equal("source-id", row.SourceResourceId); + Assert.Equal("9", row.SourceResourceVersion); + Assert.Equal("Patient.note.text", row.SourcePath); + Assert.Equal("[0.25,-1.5]", row.Embedding); + } + private static string InvokeRemoveTrailingZerosFromMillisecondsForAGivenDate(DateTimeOffset date) { var method = typeof(SqlServerFhirDataStore).GetMethod( @@ -371,7 +434,50 @@ public void GivenUnknownResourceType_WhenGettingResourceTypeId_ThenResourceNotFo Assert.Contains("is not a known resource type", exception.Message, StringComparison.Ordinal); } - private static SqlServerFhirDataStore CreateSqlServerFhirDataStore(ISqlRetryService sqlRetryService, SqlTransactionHandler sqlTransactionHandler = null) + [Theory] + [InlineData(SchemaVersionConstants.VectorSearchReindexVersion, true, "dbo.UpdateResourceSearchParamsWithVectors", true)] + [InlineData(SchemaVersionConstants.VectorSearchReindexVersion, false, "dbo.UpdateResourceSearchParams", false)] + [InlineData((int)SchemaVersion.V117, true, "dbo.UpdateResourceSearchParams", false)] + public void BulkUpdateSearchParameterIndicesAsync_SelectsProcedureForSchemaAndVectorUpdateIntent( + int schemaVersion, + bool vectorSearchIndicesUpdated, + string expectedProcedure, + bool expectedVectorParameters) + { + // Arrange + ResourceWrapper resource = CreateResourceWrapper("{\"resourceType\":\"Patient\",\"id\":\"123\"}"); + resource.ResourceSurrogateId = 42; + if (vectorSearchIndicesUpdated) + { + resource.UpdateVectorSearchIndices(Array.Empty()); + } + + // Act + bool updateVectorSearchIndices = SqlServerFhirDataStore.ShouldUpdateVectorSearchIndices(new[] { resource }, schemaVersion); + using SqlCommand command = SqlServerFhirDataStore.CreateBulkUpdateSearchParameterIndicesCommand(updateVectorSearchIndices, resourceCount: 1); + + // Assert + Assert.Equal(expectedVectorParameters, updateVectorSearchIndices); + Assert.Equal(expectedProcedure, command.CommandText); + } + + [Theory] + [InlineData((int)SchemaVersion.V119, true, true)] + [InlineData((int)SchemaVersion.V118, true, false)] + [InlineData((int)SchemaVersion.V119, false, false)] + public void SourceRefreshScheduling_SelectsOnlySupportedEnabledConfigurations( + int schemaVersion, + bool vectorSearchEnabled, + bool expected) + { + IVectorSearchIndexer vectorSearchIndexer = vectorSearchEnabled ? Substitute.For() : null; + + bool actual = SqlServerFhirDataStore.ShouldEnqueueVectorSearchSourceRefresh(vectorSearchIndexer, schemaVersion); + + Assert.Equal(expected, actual); + } + + private static SqlServerFhirDataStore CreateSqlServerFhirDataStore(ISqlRetryService sqlRetryService, SqlTransactionHandler sqlTransactionHandler = null, int? currentSchemaVersion = null) { sqlTransactionHandler ??= new SqlTransactionHandler(); @@ -379,7 +485,7 @@ private static SqlServerFhirDataStore CreateSqlServerFhirDataStore(ISqlRetryServ var schemaInfo = new SchemaInformation(SchemaVersionConstants.Min, SchemaVersionConstants.Max) { - Current = SchemaVersionConstants.Max, + Current = currentSchemaVersion ?? SchemaVersionConstants.Max, }; var searchService = Substitute.For(); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/VectorSearchSourceRefreshJob.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/VectorSearchSourceRefreshJob.cs new file mode 100644 index 0000000000..83c1a6d118 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/VectorSearchSourceRefreshJob.cs @@ -0,0 +1,110 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Operations.Reindex; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.JobManagement; +using Newtonsoft.Json; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Operations +{ + [JobTypeId((int)JobType.VectorSearchSourceRefresh)] + public class VectorSearchSourceRefreshJob : IJob + { + private readonly IVectorSearchSourceDependencyStore _dependencyStore; + private readonly Func> _fhirDataStoreFactory; + private readonly IResourceWrapperFactory _resourceWrapperFactory; + private readonly IVectorSearchIndexer _vectorSearchIndexer; + private readonly ILogger _logger; + + public VectorSearchSourceRefreshJob( + IVectorSearchSourceDependencyStore dependencyStore, + Func> fhirDataStoreFactory, + IResourceWrapperFactory resourceWrapperFactory, + ILogger logger, + IVectorSearchIndexer vectorSearchIndexer = null) + { + _dependencyStore = EnsureArg.IsNotNull(dependencyStore, nameof(dependencyStore)); + _fhirDataStoreFactory = EnsureArg.IsNotNull(fhirDataStoreFactory, nameof(fhirDataStoreFactory)); + _resourceWrapperFactory = EnsureArg.IsNotNull(resourceWrapperFactory, nameof(resourceWrapperFactory)); + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _vectorSearchIndexer = vectorSearchIndexer; + } + + public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(jobInfo, nameof(jobInfo)); + + VectorSearchSourceRefreshJobDefinition definition = JsonConvert.DeserializeObject(jobInfo.Definition) + ?? throw new InvalidOperationException("The vector search source refresh job definition is invalid."); + + if (_vectorSearchIndexer == null) + { + _logger.LogInformation("Skipping vector search source refresh job {JobId} because vector search indexing is disabled.", jobInfo.Id); + return CreateResult(0, 0); + } + + IReadOnlyCollection dependentKeys = await _dependencyStore.GetDependentResourceKeysAsync( + definition.SourceResourceType, + definition.SourceResourceId, + cancellationToken); + + if (dependentKeys.Count == 0) + { + return CreateResult(0, 0); + } + + using IScoped store = _fhirDataStoreFactory(); + IReadOnlyList resources = await store.Value.GetAsync(dependentKeys.ToList(), cancellationToken); + + if (resources.Count == 0) + { + return CreateResult(dependentKeys.Count, 0); + } + + foreach (ResourceWrapper resource in resources) + { + _resourceWrapperFactory.Update(resource); + } + + await _vectorSearchIndexer.IndexAsync(resources, cancellationToken); + + try + { + await store.Value.BulkUpdateSearchParameterIndicesAsync(resources, cancellationToken); + } + catch (PreconditionFailedException exception) + { + throw new JobExecutionSoftFailureException( + "A dependent resource changed while its vector search index was being refreshed.", + exception, + isCustomerCaused: false); + } + + return CreateResult(dependentKeys.Count, resources.Count); + } + + private static string CreateResult(int dependentResourceCount, int refreshedResourceCount) + { + return JsonConvert.SerializeObject(new Dictionary + { + ["dependentResourceCount"] = dependentResourceCount, + ["refreshedResourceCount"] = refreshedResourceCount, + }); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql new file mode 100644 index 0000000000..e775b0b487 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/117.diff.sql @@ -0,0 +1,580 @@ +/************************************************************* + Semantic search feature + Adds the EmbeddingModel registry and vector search tables. +**************************************************************/ + +IF NOT EXISTS (SELECT 1 FROM sys.tables WHERE name = 'EmbeddingModel') +BEGIN + CREATE TABLE dbo.EmbeddingModel + ( + EmbeddingModelId smallint IDENTITY(1,1) NOT NULL, + ModelName varchar(128) COLLATE Latin1_General_100_CS_AS NOT NULL, + ModelVersion varchar(64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Dimension int NOT NULL, + DistanceMetric varchar(16) COLLATE Latin1_General_100_CS_AS NOT NULL + CONSTRAINT DF_EmbeddingModel_DistanceMetric DEFAULT 'cosine', + CreatedAt datetime2(7) NOT NULL + CONSTRAINT DF_EmbeddingModel_CreatedAt DEFAULT SYSUTCDATETIME(), + CONSTRAINT PKC_EmbeddingModel PRIMARY KEY CLUSTERED (EmbeddingModelId), + CONSTRAINT U_EmbeddingModel_Name_Version UNIQUE (ModelName, ModelVersion) + ) +END +GO + +IF NOT EXISTS (SELECT 1 FROM sys.tables WHERE name = 'VectorSearchParam') +BEGIN + CREATE TABLE dbo.VectorSearchParam + ( + ResourceTypeId smallint NOT NULL, + ResourceSurrogateId bigint NOT NULL, + SearchParamId smallint NOT NULL, + ChunkOrdinal smallint NOT NULL + CONSTRAINT DF_VectorSearchParam_ChunkOrdinal DEFAULT 0, + EmbeddingModelId smallint NOT NULL, + ChunkText nvarchar(max) NOT NULL, + SourceTextHash binary(32) NOT NULL, + SourceResourceTypeId smallint NULL, + SourceResourceId varchar(64) COLLATE Latin1_General_100_CS_AS NULL, + SourceResourceVersion varchar(64) COLLATE Latin1_General_100_CS_AS NULL, + SourcePath nvarchar(512) NULL, + Embedding vector(1536) NOT NULL + ) + + ALTER TABLE dbo.VectorSearchParam SET ( LOCK_ESCALATION = AUTO ) + + ALTER TABLE dbo.VectorSearchParam ADD CONSTRAINT PKC_VectorSearchParam + PRIMARY KEY CLUSTERED + ( + ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + ChunkOrdinal + ) + WITH (DATA_COMPRESSION = PAGE) +END +GO + +IF NOT EXISTS (SELECT 1 FROM sys.table_types WHERE name = 'VectorSearchParamList') +BEGIN + CREATE TYPE dbo.VectorSearchParamList AS TABLE + ( + ResourceTypeId smallint NOT NULL, + ResourceSurrogateId bigint NOT NULL, + SearchParamId smallint NOT NULL, + ChunkOrdinal smallint NOT NULL, + EmbeddingModelId smallint NOT NULL, + ChunkText nvarchar(max) NOT NULL, + SourceTextHash binary(32) NOT NULL, + SourceResourceTypeId smallint NOT NULL, + SourceResourceId varchar(64) COLLATE Latin1_General_100_CS_AS NOT NULL, + SourceResourceVersion varchar(64) COLLATE Latin1_General_100_CS_AS NULL, + SourcePath nvarchar(512) NOT NULL, + Embedding nvarchar(max) NOT NULL + ) +END +GO + +ALTER PROCEDURE dbo.MergeResources + @AffectedRows int = 0 OUT + ,@RaiseExceptionOnConflict bit = 1 + ,@IsResourceChangeCaptureEnabled bit = 0 + ,@TransactionId bigint = NULL + ,@SingleTransaction bit = 1 + ,@Resources dbo.ResourceList READONLY + ,@ResourceWriteClaims dbo.ResourceWriteClaimList READONLY + ,@ReferenceSearchParams dbo.ReferenceSearchParamList READONLY + ,@TokenSearchParams dbo.TokenSearchParamList READONLY + ,@TokenTexts dbo.TokenTextList READONLY + ,@StringSearchParams dbo.StringSearchParamList READONLY + ,@UriSearchParams dbo.UriSearchParamList READONLY + ,@NumberSearchParams dbo.NumberSearchParamList READONLY + ,@QuantitySearchParams dbo.QuantitySearchParamList READONLY + ,@DateTimeSearchParms dbo.DateTimeSearchParamList READONLY + ,@VectorSearchParams dbo.VectorSearchParamList READONLY + ,@ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY + ,@TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY + ,@TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY + ,@TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY + ,@TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY + ,@TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +set nocount on +DECLARE @st datetime = getUTCdate() + ,@SP varchar(100) = object_name(@@procid) + ,@DummyTop bigint = 9223372036854775807 + ,@InitialTranCount int = @@trancount + ,@IsRetry bit = 0 + +DECLARE @Mode varchar(200) = isnull((SELECT 'RT=['+convert(varchar,min(ResourceTypeId))+','+convert(varchar,max(ResourceTypeId))+'] Sur=['+convert(varchar,min(ResourceSurrogateId))+','+convert(varchar,max(ResourceSurrogateId))+'] V='+convert(varchar,max(Version))+' Rows='+convert(varchar,count(*)) FROM @Resources),'Input=Empty') +SET @Mode += ' E='+convert(varchar,@RaiseExceptionOnConflict)+' CC='+convert(varchar,@IsResourceChangeCaptureEnabled)+' IT='+convert(varchar,@InitialTranCount)+' T='+isnull(convert(varchar,@TransactionId),'NULL')+' ST='+convert(varchar,@SingleTransaction) + +SET @AffectedRows = 0 + +BEGIN TRY + DECLARE @Existing AS TABLE (ResourceTypeId smallint NOT NULL, SurrogateId bigint NOT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)) + + DECLARE @ResourceInfos AS TABLE + ( + ResourceTypeId smallint NOT NULL + ,SurrogateId bigint NOT NULL + ,Version int NOT NULL + ,KeepHistory bit NOT NULL + ,PreviousVersion int NULL + ,PreviousSurrogateId bigint NULL + + PRIMARY KEY (ResourceTypeId, SurrogateId) + ) + + DECLARE @PreviousSurrogateIds AS TABLE (TypeId smallint NOT NULL, SurrogateId bigint NOT NULL PRIMARY KEY (TypeId, SurrogateId), KeepHistory bit) + + IF @InitialTranCount = 0 + BEGIN + IF EXISTS (SELECT * + FROM @Resources A JOIN dbo.Resource B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId + ) + BEGIN + BEGIN TRANSACTION + + INSERT INTO @Existing + ( ResourceTypeId, SurrogateId ) + SELECT B.ResourceTypeId, B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * FROM @Resources) A + JOIN dbo.Resource B WITH (ROWLOCK, HOLDLOCK) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0 + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + + IF @@rowcount = (SELECT count(*) FROM @Resources) SET @IsRetry = 1 + + IF @IsRetry = 0 COMMIT TRANSACTION + END + END + + SET @Mode += ' R='+convert(varchar,@IsRetry) + + IF @SingleTransaction = 1 AND @@trancount = 0 BEGIN TRANSACTION + + IF @IsRetry = 0 + BEGIN + INSERT INTO @ResourceInfos + ( ResourceTypeId, SurrogateId, Version, KeepHistory, PreviousVersion, PreviousSurrogateId ) + SELECT A.ResourceTypeId, A.ResourceSurrogateId, A.Version, A.KeepHistory, B.Version, B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * FROM @Resources WHERE HasVersionToCompare = 1) A + LEFT OUTER JOIN dbo.Resource B + ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + + IF @RaiseExceptionOnConflict = 1 AND EXISTS (SELECT * FROM @ResourceInfos WHERE (PreviousVersion IS NOT NULL AND Version <= PreviousVersion) OR (PreviousSurrogateId IS NOT NULL AND SurrogateId <= PreviousSurrogateId)) + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1 + + INSERT INTO @PreviousSurrogateIds + SELECT ResourceTypeId, PreviousSurrogateId, KeepHistory + FROM @ResourceInfos + WHERE PreviousSurrogateId IS NOT NULL + + IF @@rowcount > 0 + BEGIN + UPDATE dbo.Resource + SET IsHistory = 1 + WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId AND KeepHistory = 1) + SET @AffectedRows += @@rowcount + + IF @IsResourceChangeCaptureEnabled = 1 AND NOT EXISTS (SELECT * FROM dbo.Parameters WHERE Id = 'InvisibleHistory.IsEnabled' AND Number = 0) + UPDATE dbo.Resource + SET IsHistory = 1 + ,RawResource = 0xF + ,SearchParamHash = NULL + ,HistoryTransactionId = @TransactionId + WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId AND KeepHistory = 0) + ELSE + DELETE FROM dbo.Resource WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId AND KeepHistory = 0) + SET @AffectedRows += @@rowcount + + DELETE FROM dbo.ResourceWriteClaim WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.ReferenceSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.TokenSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.TokenText WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.StringSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.UriSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.NumberSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.QuantitySearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.DateTimeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.VectorSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.ReferenceTokenCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.TokenTokenCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.TokenDateTimeCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.TokenQuantityCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.TokenStringCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + DELETE FROM dbo.TokenNumberNumberCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount + END + + INSERT INTO dbo.Resource + ( ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash, TransactionId ) + SELECT ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash, @TransactionId + FROM @Resources + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.ResourceWriteClaim + ( ResourceSurrogateId, ClaimTypeId, ClaimValue ) + SELECT ResourceSurrogateId, ClaimTypeId, ClaimValue + FROM @ResourceWriteClaims + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.ReferenceSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion + FROM @ReferenceSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow + FROM @TokenSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenText + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, Text ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Text + FROM @TokenTexts + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.StringSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax + FROM @StringSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.UriSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri + FROM @UriSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.NumberSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue + FROM @NumberSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.QuantitySearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue + FROM @QuantitySearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.DateTimeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax + FROM @DateTimeSearchParms + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.VectorSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, Embedding ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, CAST(Embedding AS vector(1536)) + FROM @VectorSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.ReferenceTokenCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenTokenCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2 ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2 + FROM @TokenTokenCompositeSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenDateTimeCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2 ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenQuantityCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2 ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2 + FROM @TokenQuantityCompositeSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenStringCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2 ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2 + FROM @TokenStringCompositeSearchParams + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange + FROM @TokenNumberNumberCompositeSearchParams + SET @AffectedRows += @@rowcount + END + ELSE + BEGIN + INSERT INTO dbo.ResourceWriteClaim + ( ResourceSurrogateId, ClaimTypeId, ClaimValue ) + SELECT ResourceSurrogateId, ClaimTypeId, ClaimValue + FROM (SELECT TOP (@DummyTop) * FROM @ResourceWriteClaims) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.ResourceWriteClaim C WHERE C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.ReferenceSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion + FROM (SELECT TOP (@DummyTop) * FROM @ReferenceSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.ReferenceSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow + FROM (SELECT TOP (@DummyTop) * FROM @TokenSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.TokenSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenText + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, Text ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Text + FROM (SELECT TOP (@DummyTop) * FROM @TokenTexts) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.TokenText C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.StringSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax + FROM (SELECT TOP (@DummyTop) * FROM @StringSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.StringSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.UriSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri + FROM (SELECT TOP (@DummyTop) * FROM @UriSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.UriSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.NumberSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue + FROM (SELECT TOP (@DummyTop) * FROM @NumberSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.NumberSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.QuantitySearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue + FROM (SELECT TOP (@DummyTop) * FROM @QuantitySearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.QuantitySearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.DateTimeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax + FROM (SELECT TOP (@DummyTop) * FROM @DateTimeSearchParms) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.DateTimeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.VectorSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, Embedding ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, CAST(Embedding AS vector(1536)) + FROM (SELECT TOP (@DummyTop) * FROM @VectorSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.VectorSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.ReferenceTokenCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * FROM @ReferenceTokenCompositeSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.ReferenceTokenCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenTokenCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2 ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * FROM @TokenTokenCompositeSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.TokenTokenCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenDateTimeCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2 ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2 + FROM (SELECT TOP (@DummyTop) * FROM @TokenDateTimeCompositeSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.TokenDateTimeCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenQuantityCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2 ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2 + FROM (SELECT TOP (@DummyTop) * FROM @TokenQuantityCompositeSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.TokenQuantityCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenStringCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2 ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2 + FROM (SELECT TOP (@DummyTop) * FROM @TokenStringCompositeSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.TokenStringCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange + FROM (SELECT TOP (@DummyTop) * FROM @TokenNumberNumberCompositeSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.TokenNumberNumberCompositeSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + END + + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.CaptureResourceIdsForChanges @Resources + + IF @TransactionId IS NOT NULL + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId + + IF @InitialTranCount = 0 AND @@trancount > 0 COMMIT TRANSACTION + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@AffectedRows +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 AND @@trancount > 0 ROLLBACK TRANSACTION + IF error_number() = 1750 THROW + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@st; + + IF @RaiseExceptionOnConflict = 1 AND error_message() LIKE '%''dbo.Resource''%' + BEGIN + IF error_number() = 2601 + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates.', 1; + ELSE IF error_number() = 2627 + THROW 50424, 'Cannot persit resource due to a conflict with duplicated keys. Check the volume of resource being submited for ingestion.', 1; + ELSE + THROW; + END + ELSE + THROW; +END CATCH +GO + +ALTER PROCEDURE dbo.MergeResourcesAndSearchParams + @SearchParams dbo.SearchParamList READONLY + ,@ReindexId bigint = NULL + ,@IsResourceChangeCaptureEnabled bit = 0 + ,@TransactionId bigint = NULL + ,@Resources dbo.ResourceList READONLY + ,@ResourceWriteClaims dbo.ResourceWriteClaimList READONLY + ,@ReferenceSearchParams dbo.ReferenceSearchParamList READONLY + ,@TokenSearchParams dbo.TokenSearchParamList READONLY + ,@TokenTexts dbo.TokenTextList READONLY + ,@StringSearchParams dbo.StringSearchParamList READONLY + ,@UriSearchParams dbo.UriSearchParamList READONLY + ,@NumberSearchParams dbo.NumberSearchParamList READONLY + ,@QuantitySearchParams dbo.QuantitySearchParamList READONLY + ,@DateTimeSearchParms dbo.DateTimeSearchParamList READONLY + ,@VectorSearchParams dbo.VectorSearchParamList READONLY + ,@ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY + ,@TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY + ,@TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY + ,@TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY + ,@TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY + ,@TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +set nocount on +DECLARE @SP varchar(100) = object_name(@@procid) + ,@Mode varchar(200) = 'R='+convert(varchar,(SELECT count(*) FROM @Resources))+' SP='+convert(varchar,(SELECT count(*) FROM @SearchParams)) + ,@st datetime = getUTCdate() + ,@Rows int = 0 + +BEGIN TRY + SET TRANSACTION ISOLATION LEVEL SERIALIZABLE + + BEGIN TRANSACTION + + EXECUTE dbo.MergeSearchParams @SearchParams, @ReindexId + + IF EXISTS (SELECT * FROM @Resources) + EXECUTE dbo.MergeResources + @AffectedRows = @Rows OUTPUT + ,@RaiseExceptionOnConflict = 1 + ,@IsResourceChangeCaptureEnabled = @IsResourceChangeCaptureEnabled + ,@TransactionId = @TransactionId + ,@SingleTransaction = 1 + ,@Resources = @Resources + ,@ResourceWriteClaims = @ResourceWriteClaims + ,@ReferenceSearchParams = @ReferenceSearchParams + ,@TokenSearchParams = @TokenSearchParams + ,@TokenTexts = @TokenTexts + ,@StringSearchParams = @StringSearchParams + ,@UriSearchParams = @UriSearchParams + ,@NumberSearchParams = @NumberSearchParams + ,@QuantitySearchParams = @QuantitySearchParams + ,@DateTimeSearchParms = @DateTimeSearchParms + ,@VectorSearchParams = @VectorSearchParams + ,@ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams + ,@TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams + ,@TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams + ,@TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams + ,@TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams + ,@TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams; + ELSE + IF @TransactionId IS NOT NULL + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId + + COMMIT TRANSACTION + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Action='Merge',@Rows=@Rows +END TRY +BEGIN CATCH + IF @@trancount > 0 ROLLBACK TRANSACTION; + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@st; + THROW +END CATCH +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/118.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/118.diff.sql new file mode 100644 index 0000000000..0810a679a4 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/118.diff.sql @@ -0,0 +1,85 @@ +CREATE PROCEDURE dbo.UpdateResourceSearchParamsWithVectors + @FailedResources int = 0 OUT + ,@Resources dbo.ResourceList READONLY + ,@ResourceWriteClaims dbo.ResourceWriteClaimList READONLY + ,@ReferenceSearchParams dbo.ReferenceSearchParamList READONLY + ,@TokenSearchParams dbo.TokenSearchParamList READONLY + ,@TokenTexts dbo.TokenTextList READONLY + ,@StringSearchParams dbo.StringSearchParamList READONLY + ,@UriSearchParams dbo.UriSearchParamList READONLY + ,@NumberSearchParams dbo.NumberSearchParamList READONLY + ,@QuantitySearchParams dbo.QuantitySearchParamList READONLY + ,@DateTimeSearchParams dbo.DateTimeSearchParamList READONLY + ,@ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY + ,@TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY + ,@TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY + ,@TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY + ,@TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY + ,@TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY + ,@VectorSearchResources dbo.ResourceList READONLY + ,@VectorSearchParams dbo.VectorSearchParamList READONLY +AS +SET NOCOUNT ON +DECLARE @InitialTranCount int = @@TRANCOUNT + +BEGIN TRY + IF @InitialTranCount = 0 BEGIN TRANSACTION + + EXECUTE dbo.UpdateResourceSearchParams + @FailedResources = @FailedResources OUT + ,@Resources = @Resources + ,@ResourceWriteClaims = @ResourceWriteClaims + ,@ReferenceSearchParams = @ReferenceSearchParams + ,@TokenSearchParams = @TokenSearchParams + ,@TokenTexts = @TokenTexts + ,@StringSearchParams = @StringSearchParams + ,@UriSearchParams = @UriSearchParams + ,@NumberSearchParams = @NumberSearchParams + ,@QuantitySearchParams = @QuantitySearchParams + ,@DateTimeSearchParams = @DateTimeSearchParams + ,@ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams + ,@TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams + ,@TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams + ,@TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams + ,@TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams + ,@TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams + + DECLARE @Ids TABLE + ( + ResourceTypeId smallint NOT NULL, + ResourceSurrogateId bigint NOT NULL, + PRIMARY KEY (ResourceTypeId, ResourceSurrogateId) + ) + + INSERT INTO @Ids + ( ResourceTypeId, ResourceSurrogateId ) + SELECT A.ResourceTypeId, A.ResourceSurrogateId + FROM @VectorSearchResources A + JOIN dbo.Resource B WITH (UPDLOCK, HOLDLOCK) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + WHERE B.IsHistory = 0 + + DELETE V + FROM dbo.VectorSearchParam V + JOIN @Ids I + ON I.ResourceTypeId = V.ResourceTypeId + AND I.ResourceSurrogateId = V.ResourceSurrogateId + + INSERT INTO dbo.VectorSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, Embedding ) + SELECT V.ResourceTypeId, V.ResourceSurrogateId, V.SearchParamId, V.ChunkOrdinal, V.EmbeddingModelId, V.ChunkText, V.SourceTextHash, V.SourceResourceTypeId, V.SourceResourceId, V.SourceResourceVersion, V.SourcePath, CAST(V.Embedding AS vector(1536)) + FROM @VectorSearchParams V + JOIN @Ids I + ON I.ResourceTypeId = V.ResourceTypeId + AND I.ResourceSurrogateId = V.ResourceSurrogateId + + IF @InitialTranCount = 0 COMMIT TRANSACTION +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 AND @@TRANCOUNT > 0 ROLLBACK TRANSACTION + THROW +END CATCH +GO \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/119.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/119.diff.sql new file mode 100644 index 0000000000..15bf51dfc4 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/119.diff.sql @@ -0,0 +1,370 @@ +CREATE NONCLUSTERED INDEX IX_VectorSearchParam_SourceResource +ON dbo.VectorSearchParam +( + SourceResourceTypeId, + SourceResourceId +) +INCLUDE +( + ResourceTypeId, + ResourceSurrogateId +) +WHERE SourceResourceTypeId IS NOT NULL AND SourceResourceId IS NOT NULL +WITH (DATA_COMPRESSION = PAGE) +GO + +ALTER PROCEDURE dbo.HardDeleteResource + @ResourceTypeId smallint + ,@ResourceId varchar(64) + ,@KeepCurrentVersion bit + ,@IsResourceChangeCaptureEnabled bit +AS +set nocount on +DECLARE @SP varchar(100) = object_name(@@procid) + ,@Mode varchar(200) = 'RT='+convert(varchar,@ResourceTypeId)+' R='+@ResourceId+' V='+convert(varchar,@KeepCurrentVersion)+' CC='+convert(varchar,@IsResourceChangeCaptureEnabled) + ,@st datetime = getUTCdate() + ,@InitialTranCount int = @@trancount + ,@TransactionId bigint + +BEGIN TRY + IF @IsResourceChangeCaptureEnabled = 1 EXECUTE dbo.MergeResourcesBeginTransaction @Count = 1, @TransactionId = @TransactionId OUT + + IF @KeepCurrentVersion = 0 AND @InitialTranCount = 0 + BEGIN TRANSACTION + + DECLARE @SurrogateIds TABLE (ResourceSurrogateId BIGINT NOT NULL) + + IF @IsResourceChangeCaptureEnabled = 1 AND NOT EXISTS (SELECT * FROM dbo.Parameters WHERE Id = 'InvisibleHistory.IsEnabled' AND Number = 0) + UPDATE dbo.Resource + SET IsDeleted = 1 + ,RawResource = 0xF + ,SearchParamHash = NULL + ,HistoryTransactionId = @TransactionId + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 OR IsHistory = 1) + AND RawResource <> 0xF + ELSE + DELETE dbo.Resource + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 OR IsHistory = 1) + AND RawResource <> 0xF + + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.VectorSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + + IF @KeepCurrentVersion = 0 + BEGIN + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.ResourceWriteClaim B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.ReferenceSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.TokenSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.TokenText B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.StringSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.UriSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.NumberSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.QuantitySearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.DateTimeSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.ReferenceTokenCompositeSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.TokenTokenCompositeSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.TokenDateTimeCompositeSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.TokenQuantityCompositeSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.TokenStringCompositeSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.TokenNumberNumberCompositeSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + END + + IF @InitialTranCount = 0 AND @@trancount > 0 COMMIT TRANSACTION + + IF @IsResourceChangeCaptureEnabled = 1 EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 AND @@trancount > 0 ROLLBACK TRANSACTION + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@st; + THROW +END CATCH +GO + +CREATE PROCEDURE dbo.GetVectorSearchSourceDependencies + @SourceResourceTypeId smallint, + @SourceResourceId varchar(64) +AS +SET NOCOUNT ON + +SELECT DISTINCT V.ResourceTypeId, R.ResourceId +FROM dbo.VectorSearchParam V +JOIN dbo.Resource R + ON R.ResourceTypeId = V.ResourceTypeId + AND R.ResourceSurrogateId = V.ResourceSurrogateId + AND R.IsHistory = 0 + AND R.IsDeleted = 0 +WHERE V.SourceResourceTypeId = @SourceResourceTypeId + AND V.SourceResourceId = @SourceResourceId + AND (V.ResourceTypeId <> @SourceResourceTypeId OR R.ResourceId <> @SourceResourceId) +GO + +CREATE PROCEDURE dbo.EnqueueVectorSearchSourceRefreshJobs + @Resources dbo.ResourceList READONLY +AS +SET NOCOUNT ON + +DECLARE @Definitions dbo.StringList + +INSERT INTO @Definitions (String) +SELECT DISTINCT + ( + SELECT 11 AS TypeId, + RT.Name AS SourceResourceType, + A.ResourceId AS SourceResourceId, + CONVERT(varchar(64), A.Version) AS SourceResourceVersion + FOR JSON PATH, WITHOUT_ARRAY_WRAPPER + ) +FROM @Resources A +JOIN dbo.ResourceType RT ON RT.ResourceTypeId = A.ResourceTypeId +WHERE A.IsHistory = 0 + AND EXISTS + ( + SELECT 1 + FROM dbo.VectorSearchParam V + JOIN dbo.Resource R + ON R.ResourceTypeId = V.ResourceTypeId + AND R.ResourceSurrogateId = V.ResourceSurrogateId + AND R.IsHistory = 0 + AND R.IsDeleted = 0 + WHERE V.SourceResourceTypeId = A.ResourceTypeId + AND V.SourceResourceId = A.ResourceId + AND (V.ResourceTypeId <> A.ResourceTypeId OR R.ResourceId <> A.ResourceId) + ) + +IF EXISTS (SELECT 1 FROM @Definitions) + EXECUTE dbo.EnqueueJobs + @QueueType = 6, + @Definitions = @Definitions, + @ForceOneActiveJobGroup = 0, + @ReturnJobs = 0 +GO + +DECLARE @MergeResourcesDefinition nvarchar(max) = OBJECT_DEFINITION(OBJECT_ID('dbo.MergeResources')) +DECLARE @SingleTransactionPosition int = CHARINDEX('@SingleTransaction', @MergeResourcesDefinition) +DECLARE @SingleTransactionParameterEnd int = CHARINDEX(',', @MergeResourcesDefinition, @SingleTransactionPosition) + +IF @MergeResourcesDefinition IS NULL OR @SingleTransactionPosition = 0 OR @SingleTransactionParameterEnd = 0 + THROW 50000, 'Unable to add vector source refresh support to dbo.MergeResources.', 1 + +SET @MergeResourcesDefinition = STUFF( + @MergeResourcesDefinition, + @SingleTransactionParameterEnd + 1, + 0, + ' @EnqueueVectorSearchSourceRefresh bit = 0,') + +DECLARE @CommitTransactionCallPosition int = CHARINDEX('EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId', @MergeResourcesDefinition) +DECLARE @MergeResourcesCommitPosition int = CHARINDEX('IF @InitialTranCount', @MergeResourcesDefinition, @CommitTransactionCallPosition) + +IF @CommitTransactionCallPosition = 0 OR @MergeResourcesCommitPosition = 0 + THROW 50000, 'Unable to locate the commit boundary in dbo.MergeResources.', 1 + +SET @MergeResourcesDefinition = STUFF( + @MergeResourcesDefinition, + @MergeResourcesCommitPosition, + 0, + 'IF @EnqueueVectorSearchSourceRefresh = 1 + EXECUTE dbo.EnqueueVectorSearchSourceRefreshJobs @Resources = @Resources + + ') + +DECLARE @MergeResourcesCreatePosition int = CHARINDEX('CREATE PROCEDURE', @MergeResourcesDefinition) + +IF @MergeResourcesCreatePosition > 0 + SET @MergeResourcesDefinition = STUFF( + @MergeResourcesDefinition, + @MergeResourcesCreatePosition, + LEN('CREATE PROCEDURE'), + 'ALTER PROCEDURE') +ELSE IF CHARINDEX('ALTER PROCEDURE', @MergeResourcesDefinition) = 0 + THROW 50000, 'Unable to alter dbo.MergeResources because its definition has an unexpected form.', 1 + +EXECUTE sp_executesql @MergeResourcesDefinition +GO + +DECLARE @MergeResourcesAndSearchParamsDefinition nvarchar(max) = OBJECT_DEFINITION(OBJECT_ID('dbo.MergeResourcesAndSearchParams')) +DECLARE @TransactionIdPosition int = CHARINDEX('@TransactionId', @MergeResourcesAndSearchParamsDefinition) +DECLARE @TransactionIdParameterEnd int = CHARINDEX(',', @MergeResourcesAndSearchParamsDefinition, @TransactionIdPosition) + +IF @MergeResourcesAndSearchParamsDefinition IS NULL OR @TransactionIdPosition = 0 OR @TransactionIdParameterEnd = 0 + THROW 50000, 'Unable to add vector source refresh support to dbo.MergeResourcesAndSearchParams.', 1 + +SET @MergeResourcesAndSearchParamsDefinition = STUFF( + @MergeResourcesAndSearchParamsDefinition, + @TransactionIdParameterEnd + 1, + 0, + ' @EnqueueVectorSearchSourceRefresh bit = 0,') + +DECLARE @MergeResourcesAndSearchParamsCommitPosition int = CHARINDEX('COMMIT TRANSACTION', @MergeResourcesAndSearchParamsDefinition) + +IF @MergeResourcesAndSearchParamsCommitPosition = 0 + THROW 50000, 'Unable to locate the commit boundary in dbo.MergeResourcesAndSearchParams.', 1 + +SET @MergeResourcesAndSearchParamsDefinition = STUFF( + @MergeResourcesAndSearchParamsDefinition, + @MergeResourcesAndSearchParamsCommitPosition, + 0, + 'IF @EnqueueVectorSearchSourceRefresh = 1 + EXECUTE dbo.EnqueueVectorSearchSourceRefreshJobs @Resources = @Resources + + ') + +DECLARE @MergeResourcesAndSearchParamsCreatePosition int = CHARINDEX('CREATE PROCEDURE', @MergeResourcesAndSearchParamsDefinition) + +IF @MergeResourcesAndSearchParamsCreatePosition > 0 + SET @MergeResourcesAndSearchParamsDefinition = STUFF( + @MergeResourcesAndSearchParamsDefinition, + @MergeResourcesAndSearchParamsCreatePosition, + LEN('CREATE PROCEDURE'), + 'ALTER PROCEDURE') +ELSE IF CHARINDEX('ALTER PROCEDURE', @MergeResourcesAndSearchParamsDefinition) = 0 + THROW 50000, 'Unable to alter dbo.MergeResourcesAndSearchParams because its definition has an unexpected form.', 1 + +EXECUTE sp_executesql @MergeResourcesAndSearchParamsDefinition +GO + +CREATE PROCEDURE dbo.MergeResourcesWithVectorSearchSourceRefresh + @AffectedRows int = 0 OUT, + @RaiseExceptionOnConflict bit = 1, + @IsResourceChangeCaptureEnabled bit = 0, + @TransactionId bigint = NULL, + @SingleTransaction bit = 1, + @Resources dbo.ResourceList READONLY, + @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, + @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, + @TokenSearchParams dbo.TokenSearchParamList READONLY, + @TokenTexts dbo.TokenTextList READONLY, + @StringSearchParams dbo.StringSearchParamList READONLY, + @UriSearchParams dbo.UriSearchParamList READONLY, + @NumberSearchParams dbo.NumberSearchParamList READONLY, + @QuantitySearchParams dbo.QuantitySearchParamList READONLY, + @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, + @VectorSearchParams dbo.VectorSearchParamList READONLY, + @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, + @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, + @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, + @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, + @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, + @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON +EXECUTE dbo.MergeResources + @AffectedRows = @AffectedRows OUT, + @RaiseExceptionOnConflict = @RaiseExceptionOnConflict, + @IsResourceChangeCaptureEnabled = @IsResourceChangeCaptureEnabled, + @TransactionId = @TransactionId, + @SingleTransaction = @SingleTransaction, + @EnqueueVectorSearchSourceRefresh = 1, + @Resources = @Resources, + @ResourceWriteClaims = @ResourceWriteClaims, + @ReferenceSearchParams = @ReferenceSearchParams, + @TokenSearchParams = @TokenSearchParams, + @TokenTexts = @TokenTexts, + @StringSearchParams = @StringSearchParams, + @UriSearchParams = @UriSearchParams, + @NumberSearchParams = @NumberSearchParams, + @QuantitySearchParams = @QuantitySearchParams, + @DateTimeSearchParms = @DateTimeSearchParms, + @VectorSearchParams = @VectorSearchParams, + @ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams, + @TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams, + @TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams, + @TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams, + @TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams, + @TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams +GO + +CREATE PROCEDURE dbo.MergeResourcesAndSearchParamsWithVectorSearchSourceRefresh + @SearchParams dbo.SearchParamList READONLY, + @ReindexId bigint = NULL, + @IsResourceChangeCaptureEnabled bit = 0, + @TransactionId bigint = NULL, + @Resources dbo.ResourceList READONLY, + @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, + @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, + @TokenSearchParams dbo.TokenSearchParamList READONLY, + @TokenTexts dbo.TokenTextList READONLY, + @StringSearchParams dbo.StringSearchParamList READONLY, + @UriSearchParams dbo.UriSearchParamList READONLY, + @NumberSearchParams dbo.NumberSearchParamList READONLY, + @QuantitySearchParams dbo.QuantitySearchParamList READONLY, + @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, + @VectorSearchParams dbo.VectorSearchParamList READONLY, + @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, + @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, + @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, + @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, + @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, + @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON +EXECUTE dbo.MergeResourcesAndSearchParams + @SearchParams = @SearchParams, + @ReindexId = @ReindexId, + @IsResourceChangeCaptureEnabled = @IsResourceChangeCaptureEnabled, + @TransactionId = @TransactionId, + @EnqueueVectorSearchSourceRefresh = 1, + @Resources = @Resources, + @ResourceWriteClaims = @ResourceWriteClaims, + @ReferenceSearchParams = @ReferenceSearchParams, + @TokenSearchParams = @TokenSearchParams, + @TokenTexts = @TokenTexts, + @StringSearchParams = @StringSearchParams, + @UriSearchParams = @UriSearchParams, + @NumberSearchParams = @NumberSearchParams, + @QuantitySearchParams = @QuantitySearchParams, + @DateTimeSearchParms = @DateTimeSearchParms, + @VectorSearchParams = @VectorSearchParams, + @ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams, + @TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams, + @TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams, + @TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams, + @TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams, + @TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams +GO + +CREATE PROCEDURE dbo.HardDeleteResourceWithVectorSearchSourceRefresh + @ResourceTypeId smallint, + @ResourceId varchar(64), + @KeepCurrentVersion bit, + @IsResourceChangeCaptureEnabled bit +AS +SET NOCOUNT ON +DECLARE @InitialTranCount int = @@TRANCOUNT +DECLARE @ChangedSources dbo.ResourceList + +BEGIN TRY + IF @InitialTranCount = 0 BEGIN TRANSACTION + + IF @KeepCurrentVersion = 0 + INSERT INTO @ChangedSources + (ResourceTypeId, ResourceSurrogateId, ResourceId, Version, HasVersionToCompare, IsDeleted, IsHistory, KeepHistory, RawResource, IsRawResourceMetaSet, RequestMethod, SearchParamHash) + SELECT TOP (1) + ResourceTypeId, ResourceSurrogateId, ResourceId, Version + 1, 0, 1, 0, 0, 0x0, 0, NULL, NULL + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND IsHistory = 0 + ORDER BY ResourceSurrogateId DESC + + EXECUTE dbo.HardDeleteResource + @ResourceTypeId = @ResourceTypeId, + @ResourceId = @ResourceId, + @KeepCurrentVersion = @KeepCurrentVersion, + @IsResourceChangeCaptureEnabled = @IsResourceChangeCaptureEnabled + + EXECUTE dbo.EnqueueVectorSearchSourceRefreshJobs @Resources = @ChangedSources + + IF @InitialTranCount = 0 COMMIT TRANSACTION +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 AND @@TRANCOUNT > 0 ROLLBACK TRANSACTION + THROW +END CATCH +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs index 340aebb356..c43391da2e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs @@ -126,5 +126,8 @@ public enum SchemaVersion V114 = 114, V115 = 115, V116 = 116, + V117 = 117, + V118 = 118, + V119 = 119, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index 854d345f6b..514a10aff0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema public static class SchemaVersionConstants { public const int Min = (int)SchemaVersion.V113; - public const int Max = (int)SchemaVersion.V116; + public const int Max = (int)SchemaVersion.V119; public const int MinForUpgrade = (int)SchemaVersion.V111; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; @@ -38,6 +38,8 @@ public static class SchemaVersionConstants public const int SearchParameterMaxLastUpdatedStoredProcedure = (int)SchemaVersion.V96; public const int SearchParameterLastUpdatedIndex = (int)SchemaVersion.V98; public const int FhirModelInitialization = (int)SchemaVersion.V107; + public const int VectorSearchReindexVersion = (int)SchemaVersion.V118; + public const int VectorSearchSourceRefreshVersion = (int)SchemaVersion.V119; // It is currently used in Azure Healthcare APIs. public const int ParameterizedRemovePartitionFromResourceChangesVersion = (int)SchemaVersion.V21; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/EnqueueVectorSearchSourceRefreshJobs.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/EnqueueVectorSearchSourceRefreshJobs.sql new file mode 100644 index 0000000000..3408b6cd1e --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/EnqueueVectorSearchSourceRefreshJobs.sql @@ -0,0 +1,40 @@ +CREATE PROCEDURE dbo.EnqueueVectorSearchSourceRefreshJobs + @Resources dbo.ResourceList READONLY +AS +SET NOCOUNT ON + +DECLARE @Definitions dbo.StringList + +INSERT INTO @Definitions (String) +SELECT DISTINCT + ( + SELECT 11 AS TypeId, + RT.Name AS SourceResourceType, + A.ResourceId AS SourceResourceId, + CONVERT(varchar(64), A.Version) AS SourceResourceVersion + FOR JSON PATH, WITHOUT_ARRAY_WRAPPER + ) +FROM @Resources A +JOIN dbo.ResourceType RT ON RT.ResourceTypeId = A.ResourceTypeId +WHERE A.IsHistory = 0 + AND EXISTS + ( + SELECT 1 + FROM dbo.VectorSearchParam V + JOIN dbo.Resource R + ON R.ResourceTypeId = V.ResourceTypeId + AND R.ResourceSurrogateId = V.ResourceSurrogateId + AND R.IsHistory = 0 + AND R.IsDeleted = 0 + WHERE V.SourceResourceTypeId = A.ResourceTypeId + AND V.SourceResourceId = A.ResourceId + AND (V.ResourceTypeId <> A.ResourceTypeId OR R.ResourceId <> A.ResourceId) + ) + +IF EXISTS (SELECT 1 FROM @Definitions) + EXECUTE dbo.EnqueueJobs + @QueueType = 6, + @Definitions = @Definitions, + @ForceOneActiveJobGroup = 0, + @ReturnJobs = 0 +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetVectorSearchSourceDependencies.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetVectorSearchSourceDependencies.sql new file mode 100644 index 0000000000..0f9fad67d8 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetVectorSearchSourceDependencies.sql @@ -0,0 +1,17 @@ +CREATE PROCEDURE dbo.GetVectorSearchSourceDependencies + @SourceResourceTypeId smallint, + @SourceResourceId varchar(64) +AS +SET NOCOUNT ON + +SELECT DISTINCT V.ResourceTypeId, R.ResourceId +FROM dbo.VectorSearchParam V +JOIN dbo.Resource R + ON R.ResourceTypeId = V.ResourceTypeId + AND R.ResourceSurrogateId = V.ResourceSurrogateId + AND R.IsHistory = 0 + AND R.IsDeleted = 0 +WHERE V.SourceResourceTypeId = @SourceResourceTypeId + AND V.SourceResourceId = @SourceResourceId + AND (V.ResourceTypeId <> @SourceResourceTypeId OR R.ResourceId <> @SourceResourceId) +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/HardDeleteResource.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/HardDeleteResource.sql index 6b85ea2747..8bc96c3ca4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/HardDeleteResource.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/HardDeleteResource.sql @@ -8,12 +8,13 @@ set nocount on DECLARE @SP varchar(100) = object_name(@@procid) ,@Mode varchar(200) = 'RT='+convert(varchar,@ResourceTypeId)+' R='+@ResourceId+' V='+convert(varchar,@KeepCurrentVersion)+' CC='+convert(varchar,@IsResourceChangeCaptureEnabled) ,@st datetime = getUTCdate() + ,@InitialTranCount int = @@trancount ,@TransactionId bigint BEGIN TRY IF @IsResourceChangeCaptureEnabled = 1 EXECUTE dbo.MergeResourcesBeginTransaction @Count = 1, @TransactionId = @TransactionId OUT - IF @KeepCurrentVersion = 0 + IF @KeepCurrentVersion = 0 AND @InitialTranCount = 0 BEGIN TRANSACTION DECLARE @SurrogateIds TABLE (ResourceSurrogateId BIGINT NOT NULL) @@ -37,6 +38,8 @@ BEGIN TRY AND (@KeepCurrentVersion = 0 OR IsHistory = 1) AND RawResource <> 0xF + DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.VectorSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) + IF @KeepCurrentVersion = 0 BEGIN -- PAGLOCK allows deallocation of empty page without waiting for ghost cleanup @@ -57,14 +60,14 @@ BEGIN TRY DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.TokenNumberNumberCompositeSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1) END - IF @@trancount > 0 COMMIT TRANSACTION + IF @InitialTranCount = 0 AND @@trancount > 0 COMMIT TRANSACTION IF @IsResourceChangeCaptureEnabled = 1 EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st END TRY BEGIN CATCH - IF @@trancount > 0 ROLLBACK TRANSACTION + IF @InitialTranCount = 0 AND @@trancount > 0 ROLLBACK TRANSACTION EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@st; THROW END CATCH diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/HardDeleteResourceWithVectorSearchSourceRefresh.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/HardDeleteResourceWithVectorSearchSourceRefresh.sql new file mode 100644 index 0000000000..c9a759a6b1 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/HardDeleteResourceWithVectorSearchSourceRefresh.sql @@ -0,0 +1,39 @@ +CREATE PROCEDURE dbo.HardDeleteResourceWithVectorSearchSourceRefresh + @ResourceTypeId smallint, + @ResourceId varchar(64), + @KeepCurrentVersion bit, + @IsResourceChangeCaptureEnabled bit +AS +SET NOCOUNT ON +DECLARE @InitialTranCount int = @@TRANCOUNT +DECLARE @ChangedSources dbo.ResourceList + +BEGIN TRY + IF @InitialTranCount = 0 BEGIN TRANSACTION + + IF @KeepCurrentVersion = 0 + INSERT INTO @ChangedSources + (ResourceTypeId, ResourceSurrogateId, ResourceId, Version, HasVersionToCompare, IsDeleted, IsHistory, KeepHistory, RawResource, IsRawResourceMetaSet, RequestMethod, SearchParamHash) + SELECT TOP (1) + ResourceTypeId, ResourceSurrogateId, ResourceId, Version + 1, 0, 1, 0, 0, 0x0, 0, NULL, NULL + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND IsHistory = 0 + ORDER BY ResourceSurrogateId DESC + + EXECUTE dbo.HardDeleteResource + @ResourceTypeId = @ResourceTypeId, + @ResourceId = @ResourceId, + @KeepCurrentVersion = @KeepCurrentVersion, + @IsResourceChangeCaptureEnabled = @IsResourceChangeCaptureEnabled + + EXECUTE dbo.EnqueueVectorSearchSourceRefreshJobs @Resources = @ChangedSources + + IF @InitialTranCount = 0 COMMIT TRANSACTION +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 AND @@TRANCOUNT > 0 ROLLBACK TRANSACTION + THROW +END CATCH +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResources.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResources.sql index 61d536fa6a..9e6553a5bd 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResources.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResources.sql @@ -10,6 +10,7 @@ CREATE PROCEDURE dbo.MergeResources ,@IsResourceChangeCaptureEnabled bit = 0 ,@TransactionId bigint = NULL ,@SingleTransaction bit = 1 + ,@EnqueueVectorSearchSourceRefresh bit = 0 ,@Resources dbo.ResourceList READONLY ,@ResourceWriteClaims dbo.ResourceWriteClaimList READONLY ,@ReferenceSearchParams dbo.ReferenceSearchParamList READONLY @@ -20,6 +21,7 @@ CREATE PROCEDURE dbo.MergeResources ,@NumberSearchParams dbo.NumberSearchParamList READONLY ,@QuantitySearchParams dbo.QuantitySearchParamList READONLY ,@DateTimeSearchParms dbo.DateTimeSearchParamList READONLY + ,@VectorSearchParams dbo.VectorSearchParamList READONLY ,@ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY ,@TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY ,@TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY @@ -143,6 +145,8 @@ BEGIN TRY SET @AffectedRows += @@rowcount DELETE FROM dbo.DateTimeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) SET @AffectedRows += @@rowcount + DELETE FROM dbo.VectorSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) + SET @AffectedRows += @@rowcount DELETE FROM dbo.ReferenceTokenCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) SET @AffectedRows += @@rowcount DELETE FROM dbo.TokenTokenCompositeSearchParam WHERE EXISTS (SELECT * FROM @PreviousSurrogateIds WHERE TypeId = ResourceTypeId AND SurrogateId = ResourceSurrogateId) @@ -219,6 +223,12 @@ BEGIN TRY FROM @DateTimeSearchParms SET @AffectedRows += @@rowcount + INSERT INTO dbo.VectorSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, Embedding ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, CAST(Embedding AS vector(1536)) + FROM @VectorSearchParams + SET @AffectedRows += @@rowcount + INSERT INTO dbo.ReferenceTokenCompositeSearchParam ( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 ) SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 @@ -338,6 +348,15 @@ BEGIN TRY OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) SET @AffectedRows += @@rowcount + INSERT INTO dbo.VectorSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, Embedding ) + SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, CAST(Embedding AS vector(1536)) + FROM (SELECT TOP (@DummyTop) * FROM @VectorSearchParams) A + WHERE EXISTS (SELECT * FROM @Existing B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * FROM dbo.VectorSearchParam C WHERE C.ResourceTypeId = A.ResourceTypeId AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + SET @AffectedRows += @@rowcount + INSERT INTO dbo.ReferenceTokenCompositeSearchParam ( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 ) SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 @@ -399,6 +418,9 @@ BEGIN TRY IF @TransactionId IS NOT NULL EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId + IF @EnqueueVectorSearchSourceRefresh = 1 + EXECUTE dbo.EnqueueVectorSearchSourceRefreshJobs @Resources = @Resources + IF @InitialTranCount = 0 AND @@trancount > 0 COMMIT TRANSACTION EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@AffectedRows diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResourcesAndSearchParams.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResourcesAndSearchParams.sql index 54481112ce..9a8b6e27bc 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResourcesAndSearchParams.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResourcesAndSearchParams.sql @@ -3,6 +3,7 @@ ,@ReindexId bigint = NULL ,@IsResourceChangeCaptureEnabled bit = 0 ,@TransactionId bigint = NULL + ,@EnqueueVectorSearchSourceRefresh bit = 0 ,@Resources dbo.ResourceList READONLY ,@ResourceWriteClaims dbo.ResourceWriteClaimList READONLY ,@ReferenceSearchParams dbo.ReferenceSearchParamList READONLY @@ -13,6 +14,7 @@ ,@NumberSearchParams dbo.NumberSearchParamList READONLY ,@QuantitySearchParams dbo.QuantitySearchParamList READONLY ,@DateTimeSearchParms dbo.DateTimeSearchParamList READONLY + ,@VectorSearchParams dbo.VectorSearchParamList READONLY ,@ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY ,@TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY ,@TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY @@ -50,6 +52,7 @@ BEGIN TRY ,@NumberSearchParams = @NumberSearchParams ,@QuantitySearchParams = @QuantitySearchParams ,@DateTimeSearchParms = @DateTimeSearchParms + ,@VectorSearchParams = @VectorSearchParams ,@ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams ,@TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams ,@TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams @@ -60,6 +63,9 @@ BEGIN TRY IF @TransactionId IS NOT NULL EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId + IF @EnqueueVectorSearchSourceRefresh = 1 + EXECUTE dbo.EnqueueVectorSearchSourceRefreshJobs @Resources = @Resources + COMMIT TRANSACTION EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Action='Merge',@Rows=@Rows diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResourcesAndSearchParamsWithVectorSearchSourceRefresh.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResourcesAndSearchParamsWithVectorSearchSourceRefresh.sql new file mode 100644 index 0000000000..b65b31cdb1 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResourcesAndSearchParamsWithVectorSearchSourceRefresh.sql @@ -0,0 +1,48 @@ +CREATE PROCEDURE dbo.MergeResourcesAndSearchParamsWithVectorSearchSourceRefresh + @SearchParams dbo.SearchParamList READONLY, + @ReindexId bigint = NULL, + @IsResourceChangeCaptureEnabled bit = 0, + @TransactionId bigint = NULL, + @Resources dbo.ResourceList READONLY, + @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, + @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, + @TokenSearchParams dbo.TokenSearchParamList READONLY, + @TokenTexts dbo.TokenTextList READONLY, + @StringSearchParams dbo.StringSearchParamList READONLY, + @UriSearchParams dbo.UriSearchParamList READONLY, + @NumberSearchParams dbo.NumberSearchParamList READONLY, + @QuantitySearchParams dbo.QuantitySearchParamList READONLY, + @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, + @VectorSearchParams dbo.VectorSearchParamList READONLY, + @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, + @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, + @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, + @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, + @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, + @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON +EXECUTE dbo.MergeResourcesAndSearchParams + @SearchParams = @SearchParams, + @ReindexId = @ReindexId, + @IsResourceChangeCaptureEnabled = @IsResourceChangeCaptureEnabled, + @TransactionId = @TransactionId, + @EnqueueVectorSearchSourceRefresh = 1, + @Resources = @Resources, + @ResourceWriteClaims = @ResourceWriteClaims, + @ReferenceSearchParams = @ReferenceSearchParams, + @TokenSearchParams = @TokenSearchParams, + @TokenTexts = @TokenTexts, + @StringSearchParams = @StringSearchParams, + @UriSearchParams = @UriSearchParams, + @NumberSearchParams = @NumberSearchParams, + @QuantitySearchParams = @QuantitySearchParams, + @DateTimeSearchParms = @DateTimeSearchParms, + @VectorSearchParams = @VectorSearchParams, + @ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams, + @TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams, + @TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams, + @TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams, + @TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams, + @TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResourcesWithVectorSearchSourceRefresh.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResourcesWithVectorSearchSourceRefresh.sql new file mode 100644 index 0000000000..45d4fe8db2 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResourcesWithVectorSearchSourceRefresh.sql @@ -0,0 +1,50 @@ +CREATE PROCEDURE dbo.MergeResourcesWithVectorSearchSourceRefresh + @AffectedRows int = 0 OUT, + @RaiseExceptionOnConflict bit = 1, + @IsResourceChangeCaptureEnabled bit = 0, + @TransactionId bigint = NULL, + @SingleTransaction bit = 1, + @Resources dbo.ResourceList READONLY, + @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, + @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, + @TokenSearchParams dbo.TokenSearchParamList READONLY, + @TokenTexts dbo.TokenTextList READONLY, + @StringSearchParams dbo.StringSearchParamList READONLY, + @UriSearchParams dbo.UriSearchParamList READONLY, + @NumberSearchParams dbo.NumberSearchParamList READONLY, + @QuantitySearchParams dbo.QuantitySearchParamList READONLY, + @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, + @VectorSearchParams dbo.VectorSearchParamList READONLY, + @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, + @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, + @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, + @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, + @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, + @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON +EXECUTE dbo.MergeResources + @AffectedRows = @AffectedRows OUT, + @RaiseExceptionOnConflict = @RaiseExceptionOnConflict, + @IsResourceChangeCaptureEnabled = @IsResourceChangeCaptureEnabled, + @TransactionId = @TransactionId, + @SingleTransaction = @SingleTransaction, + @EnqueueVectorSearchSourceRefresh = 1, + @Resources = @Resources, + @ResourceWriteClaims = @ResourceWriteClaims, + @ReferenceSearchParams = @ReferenceSearchParams, + @TokenSearchParams = @TokenSearchParams, + @TokenTexts = @TokenTexts, + @StringSearchParams = @StringSearchParams, + @UriSearchParams = @UriSearchParams, + @NumberSearchParams = @NumberSearchParams, + @QuantitySearchParams = @QuantitySearchParams, + @DateTimeSearchParms = @DateTimeSearchParms, + @VectorSearchParams = @VectorSearchParams, + @ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams, + @TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams, + @TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams, + @TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams, + @TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams, + @TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/UpdateResourceSearchParamsWithVectors.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/UpdateResourceSearchParamsWithVectors.sql new file mode 100644 index 0000000000..0810a679a4 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/UpdateResourceSearchParamsWithVectors.sql @@ -0,0 +1,85 @@ +CREATE PROCEDURE dbo.UpdateResourceSearchParamsWithVectors + @FailedResources int = 0 OUT + ,@Resources dbo.ResourceList READONLY + ,@ResourceWriteClaims dbo.ResourceWriteClaimList READONLY + ,@ReferenceSearchParams dbo.ReferenceSearchParamList READONLY + ,@TokenSearchParams dbo.TokenSearchParamList READONLY + ,@TokenTexts dbo.TokenTextList READONLY + ,@StringSearchParams dbo.StringSearchParamList READONLY + ,@UriSearchParams dbo.UriSearchParamList READONLY + ,@NumberSearchParams dbo.NumberSearchParamList READONLY + ,@QuantitySearchParams dbo.QuantitySearchParamList READONLY + ,@DateTimeSearchParams dbo.DateTimeSearchParamList READONLY + ,@ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY + ,@TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY + ,@TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY + ,@TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY + ,@TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY + ,@TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY + ,@VectorSearchResources dbo.ResourceList READONLY + ,@VectorSearchParams dbo.VectorSearchParamList READONLY +AS +SET NOCOUNT ON +DECLARE @InitialTranCount int = @@TRANCOUNT + +BEGIN TRY + IF @InitialTranCount = 0 BEGIN TRANSACTION + + EXECUTE dbo.UpdateResourceSearchParams + @FailedResources = @FailedResources OUT + ,@Resources = @Resources + ,@ResourceWriteClaims = @ResourceWriteClaims + ,@ReferenceSearchParams = @ReferenceSearchParams + ,@TokenSearchParams = @TokenSearchParams + ,@TokenTexts = @TokenTexts + ,@StringSearchParams = @StringSearchParams + ,@UriSearchParams = @UriSearchParams + ,@NumberSearchParams = @NumberSearchParams + ,@QuantitySearchParams = @QuantitySearchParams + ,@DateTimeSearchParams = @DateTimeSearchParams + ,@ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams + ,@TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams + ,@TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams + ,@TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams + ,@TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams + ,@TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams + + DECLARE @Ids TABLE + ( + ResourceTypeId smallint NOT NULL, + ResourceSurrogateId bigint NOT NULL, + PRIMARY KEY (ResourceTypeId, ResourceSurrogateId) + ) + + INSERT INTO @Ids + ( ResourceTypeId, ResourceSurrogateId ) + SELECT A.ResourceTypeId, A.ResourceSurrogateId + FROM @VectorSearchResources A + JOIN dbo.Resource B WITH (UPDLOCK, HOLDLOCK) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + WHERE B.IsHistory = 0 + + DELETE V + FROM dbo.VectorSearchParam V + JOIN @Ids I + ON I.ResourceTypeId = V.ResourceTypeId + AND I.ResourceSurrogateId = V.ResourceSurrogateId + + INSERT INTO dbo.VectorSearchParam + ( ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, Embedding ) + SELECT V.ResourceTypeId, V.ResourceSurrogateId, V.SearchParamId, V.ChunkOrdinal, V.EmbeddingModelId, V.ChunkText, V.SourceTextHash, V.SourceResourceTypeId, V.SourceResourceId, V.SourceResourceVersion, V.SourcePath, CAST(V.Embedding AS vector(1536)) + FROM @VectorSearchParams V + JOIN @Ids I + ON I.ResourceTypeId = V.ResourceTypeId + AND I.ResourceSurrogateId = V.ResourceSurrogateId + + IF @InitialTranCount = 0 COMMIT TRANSACTION +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 AND @@TRANCOUNT > 0 ROLLBACK TRANSACTION + THROW +END CATCH +GO \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Tables/EmbeddingModel.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Tables/EmbeddingModel.sql new file mode 100644 index 0000000000..bf129d4213 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Tables/EmbeddingModel.sql @@ -0,0 +1,14 @@ + +CREATE TABLE dbo.EmbeddingModel +( + EmbeddingModelId smallint IDENTITY(1,1) NOT NULL, + ModelName varchar(128) COLLATE Latin1_General_100_CS_AS NOT NULL, + ModelVersion varchar(64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Dimension int NOT NULL, + DistanceMetric varchar(16) COLLATE Latin1_General_100_CS_AS NOT NULL + CONSTRAINT DF_EmbeddingModel_DistanceMetric DEFAULT 'cosine', + CreatedAt datetime2(7) NOT NULL + CONSTRAINT DF_EmbeddingModel_CreatedAt DEFAULT SYSUTCDATETIME(), + CONSTRAINT PKC_EmbeddingModel PRIMARY KEY CLUSTERED (EmbeddingModelId), + CONSTRAINT U_EmbeddingModel_Name_Version UNIQUE (ModelName, ModelVersion) +) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Tables/VectorSearchParam.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Tables/VectorSearchParam.sql new file mode 100644 index 0000000000..e67a89df1f --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Tables/VectorSearchParam.sql @@ -0,0 +1,42 @@ +CREATE TABLE dbo.VectorSearchParam +( + ResourceTypeId smallint NOT NULL, + ResourceSurrogateId bigint NOT NULL, + SearchParamId smallint NOT NULL, + ChunkOrdinal smallint NOT NULL + CONSTRAINT DF_VectorSearchParam_ChunkOrdinal DEFAULT 0, + EmbeddingModelId smallint NOT NULL, + ChunkText nvarchar(max) NOT NULL, + SourceTextHash binary(32) NOT NULL, + SourceResourceTypeId smallint NULL, + SourceResourceId varchar(64) COLLATE Latin1_General_100_CS_AS NULL, + SourceResourceVersion varchar(64) COLLATE Latin1_General_100_CS_AS NULL, + SourcePath nvarchar(512) NULL, + Embedding vector(1536) NOT NULL +) + +ALTER TABLE dbo.VectorSearchParam SET ( LOCK_ESCALATION = AUTO ) + +ALTER TABLE dbo.VectorSearchParam ADD CONSTRAINT PKC_VectorSearchParam +PRIMARY KEY CLUSTERED +( + ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + ChunkOrdinal +) +WITH (DATA_COMPRESSION = PAGE) + +CREATE NONCLUSTERED INDEX IX_VectorSearchParam_SourceResource +ON dbo.VectorSearchParam +( + SourceResourceTypeId, + SourceResourceId +) +INCLUDE +( + ResourceTypeId, + ResourceSurrogateId +) +WHERE SourceResourceTypeId IS NOT NULL AND SourceResourceId IS NOT NULL +WITH (DATA_COMPRESSION = PAGE) \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Types/VectorSearchParamList.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Types/VectorSearchParamList.sql new file mode 100644 index 0000000000..7e30151547 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Types/VectorSearchParamList.sql @@ -0,0 +1,18 @@ +--DROP TYPE dbo.VectorSearchParamList +GO +CREATE TYPE dbo.VectorSearchParamList AS TABLE +( + ResourceTypeId smallint NOT NULL + ,ResourceSurrogateId bigint NOT NULL + ,SearchParamId smallint NOT NULL + ,ChunkOrdinal smallint NOT NULL + ,EmbeddingModelId smallint NOT NULL + ,ChunkText nvarchar(max) NOT NULL + ,SourceTextHash binary(32) NOT NULL + ,SourceResourceTypeId smallint NOT NULL + ,SourceResourceId varchar(64) COLLATE Latin1_General_100_CS_AS NOT NULL + ,SourceResourceVersion varchar(64) COLLATE Latin1_General_100_CS_AS NULL + ,SourcePath nvarchar(512) NOT NULL + ,Embedding nvarchar(max) NOT NULL +) +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/ContinuationToken.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/ContinuationToken.cs index 511c8bd571..c5504ac08e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/ContinuationToken.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/ContinuationToken.cs @@ -67,6 +67,37 @@ public string SortValue } } + internal bool TryGetSemanticCursor(out double distance, out short resourceTypeId, out long resourceSurrogateId) + { + distance = default; + resourceTypeId = default; + resourceSurrogateId = default; + + if (_tokens.Length != 3 || + _tokens[0] is not string distanceText || + !double.TryParse(distanceText, NumberStyles.Float, CultureInfo.InvariantCulture, out distance) || + !double.IsFinite(distance) || + _tokens[2] is not long parsedResourceSurrogateId) + { + return false; + } + + resourceTypeId = _tokens[1] switch + { + short value => value, + long value when value >= short.MinValue && value <= short.MaxValue => (short)value, + _ => default, + }; + + if (resourceTypeId == default) + { + return false; + } + + resourceSurrogateId = parsedResourceSurrogateId; + return true; + } + public string ToJson() { return JsonSerializer.Serialize(_tokens); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index fa8247e193..b41b88bd9e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -11,13 +11,16 @@ using EnsureThat; using Microsoft.Data.SqlClient; using Microsoft.Health.Fhir.Api.Features.Filters; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.SqlServer.Features.Schema; using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; +using Microsoft.Health.Fhir.SqlServer.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.SqlServer; using Microsoft.Health.SqlServer.Features.Schema; @@ -111,6 +114,8 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions } _rootExpression = expression; + SqlSearchOptions sqlSearchOptions = context as SqlSearchOptions; + bool isVectorSearch = sqlSearchOptions?.PreparedVectorQuery != null; // Fail-closed invariant: when a SMART compartment membership context was attached for this search // (see SqlServerSearchService.AttachSmartCompartmentMembership), it must still be present on the @@ -252,7 +257,12 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions if (searchOptions.CountOnly) { - if (expression.SearchParamTableExpressions.Count > 0) + if (isVectorSearch) + { + selectingFromResourceTable = true; + StringBuilder.Append("SELECT count_big(DISTINCT ").Append(VLatest.Resource.ResourceSurrogateId, resourceTableAlias).AppendLine(")"); + } + else if (expression.SearchParamTableExpressions.Count > 0) { // The last CTE has all the surrogate IDs that match the results. // We just need to count those and don't need to join with the Resource table @@ -274,7 +284,7 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions // to ensure pagination works correctly. Previously TOP was in the inner subquery without ORDER BY, // causing SQL Server to return arbitrary rows before the outer ORDER BY reordered them. // Fix for pagination bug introduced in commit 6dd540c7d. - if (expression.SearchParamTableExpressions.Count == 0) + if (expression.SearchParamTableExpressions.Count == 0 || isVectorSearch) { StringBuilder.Append("SELECT TOP (").Append(Parameters.AddParameter(context.MaxItemCount + 1, includeInHash: false)).Append(") * FROM ("); } @@ -293,9 +303,11 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions .Append(VLatest.Resource.ResourceSurrogateId, resourceTableAlias).Append(", ") .Append(VLatest.Resource.RequestMethod, resourceTableAlias).Append(", "); - // If there's a table expression, use the previously selected bit, otherwise everything in the select is considered a match - StringBuilder.Append(expression.SearchParamTableExpressions.Count > 0 ? "CAST(IsMatch AS bit) AS IsMatch, " : "CAST(1 AS bit) AS IsMatch, "); - StringBuilder.Append(expression.SearchParamTableExpressions.Count > 0 ? "CAST(IsPartial AS bit) AS IsPartial, " : "CAST(0 AS bit) AS IsPartial, "); + // If there's a table expression, use the previously selected bit, otherwise everything in the select is considered a match. + // Vector search suppresses the Top CTE that carries IsMatch/IsPartial, and every ranked row is a match. + bool selectMatchBitFromCte = expression.SearchParamTableExpressions.Count > 0 && !isVectorSearch; + StringBuilder.Append(selectMatchBitFromCte ? "CAST(IsMatch AS bit) AS IsMatch, " : "CAST(1 AS bit) AS IsMatch, "); + StringBuilder.Append(selectMatchBitFromCte ? "CAST(IsPartial AS bit) AS IsPartial, " : "CAST(0 AS bit) AS IsPartial, "); StringBuilder.Append(VLatest.Resource.IsRawResourceMetaSet, resourceTableAlias).Append(", "); @@ -306,6 +318,11 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions StringBuilder.Append(VLatest.Resource.RawResource, resourceTableAlias); + if (isVectorSearch) + { + StringBuilder.Append(", semantic.SemanticDistance, semantic.SemanticChunkOrdinal, semantic.SemanticChunkText, semantic.SemanticSourceResourceTypeId, semantic.SemanticSourceResourceId, semantic.SemanticSourceResourceVersion, semantic.SemanticSourcePath, semantic.SemanticEvidenceJson"); + } + if (IsSortValueNeeded(context) && !context.IsIncludesOperation) { StringBuilder.Append(", ").Append(TableExpressionName(_tableExpressionCounter)).Append(".SortValue"); @@ -341,6 +358,11 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions .Append(VLatest.Resource.ResourceSurrogateId, resourceTableAlias).Append(" = ").Append(TableExpressionName(_tableExpressionCounter)).AppendLine(".Sid1"); } + if (isVectorSearch) + { + AppendVectorSearchApply(sqlSearchOptions.PreparedVectorQuery, resourceTableAlias); + } + using (var delimitedClause = StringBuilder.BeginDelimitedWhereClause()) { foreach (var denormalizedPredicate in expression.ResourceTableExpressions) @@ -352,6 +374,22 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions AppendHistoryClause(delimitedClause, context.ResourceVersionTypes); AppendDeletedClause(delimitedClause, context.ResourceVersionTypes); + + if (isVectorSearch && sqlSearchOptions.SemanticContinuationDistance.HasValue) + { + object distanceParameter = Parameters.AddParameter(sqlSearchOptions.SemanticContinuationDistance.Value, includeInHash: false); + object resourceTypeParameter = Parameters.AddParameter(sqlSearchOptions.SemanticContinuationResourceTypeId.Value, includeInHash: false); + object surrogateIdParameter = Parameters.AddParameter(sqlSearchOptions.SemanticContinuationResourceSurrogateId.Value, includeInHash: false); + + delimitedClause.BeginDelimitedElement(); + StringBuilder + .Append("(semantic.SemanticDistance > ").Append(distanceParameter) + .Append(" OR (semantic.SemanticDistance = ").Append(distanceParameter).Append(" AND (") + .Append(VLatest.Resource.ResourceTypeId, resourceTableAlias).Append(" > ").Append(resourceTypeParameter) + .Append(" OR (").Append(VLatest.Resource.ResourceTypeId, resourceTableAlias).Append(" = ").Append(resourceTypeParameter) + .Append(" AND ").Append(VLatest.Resource.ResourceSurrogateId, resourceTableAlias).Append(" > ").Append(surrogateIdParameter) + .Append("))))"); + } } if (!searchOptions.CountOnly) @@ -367,7 +405,14 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions StringBuilder.Append("IsMatch DESC, "); } - if (IsPrimaryKeySort(searchOptions)) + if (isVectorSearch && (searchOptions.Sort.Count == 0 || IsScoreSort(searchOptions))) + { + StringBuilder + .Append("SemanticDistance ASC, ") + .Append(VLatest.Resource.ResourceTypeId, orderTableAlias).Append(" ASC, ") + .Append(VLatest.Resource.ResourceSurrogateId, orderTableAlias).AppendLine(" ASC "); + } + else if (IsPrimaryKeySort(searchOptions)) { StringBuilder.AppendDelimited(", ", searchOptions.Sort, (sb, sort) => { @@ -441,6 +486,164 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions return null; } + private void AppendVectorSearchApply(PreparedVectorSearchQuery preparedQuery, string resourceTableAlias) + { + const string vectorTableAlias = "v"; + const string evidenceTableAlias = "ev"; + const string referenceTableAlias = "semanticReference"; + const string witnessTableAlias = "semanticWitness"; + PreparedVectorSearchChainLink chainLink = null; + if (preparedQuery.ChainLinks.Count > 0) + { + if (preparedQuery.ChainLinks.Count != 1) + { + throw new InvalidSearchOperationException("Semantic search currently supports one chain relationship."); + } + + chainLink = preparedQuery.ChainLinks[0]; + } + + short searchParamId = Model.GetSearchParamId(preparedQuery.SearchParameter.Url); + _searchParamIds.Add(searchParamId); + object distanceMetricParameter = Parameters.AddParameter(VectorSearchConfiguration.SupportedDistanceMetric, includeInHash: false); + object queryEmbeddingParameter = Parameters.AddParameter(SqlVectorFormatter.Format(preparedQuery.Embedding), includeInHash: false); + object maximumDistanceParameter = Parameters.AddParameter(2 * (1 - preparedQuery.MinimumScore), includeInHash: false); + + StringBuilder.AppendLine(" CROSS APPLY") + .AppendLine(" (") + .Append(" SELECT TOP (1) VECTOR_DISTANCE(").Append(distanceMetricParameter).Append(", ") + .Append(vectorTableAlias).Append('.').Append(VLatest.VectorSearchParamTable.Embedding).Append(", CAST(") + .Append(queryEmbeddingParameter).Append(" AS VECTOR(").Append(VectorSearchConfiguration.SupportedDimensions).AppendLine("))) AS SemanticDistance,") + .Append(" ").Append(VLatest.VectorSearchParam.ChunkOrdinal, vectorTableAlias).AppendLine(" AS SemanticChunkOrdinal,") + .Append(" ").Append(VLatest.VectorSearchParam.ChunkText, vectorTableAlias).AppendLine(" AS SemanticChunkText,") + .Append(" ").Append(VLatest.VectorSearchParam.SourceResourceTypeId, vectorTableAlias).AppendLine(" AS SemanticSourceResourceTypeId,") + .Append(" ").Append(VLatest.VectorSearchParam.SourceResourceId, vectorTableAlias).AppendLine(" AS SemanticSourceResourceId,") + .Append(" ").Append(VLatest.VectorSearchParam.SourceResourceVersion, vectorTableAlias).AppendLine(" AS SemanticSourceResourceVersion,") + .Append(" ").Append(VLatest.VectorSearchParam.SourcePath, vectorTableAlias).AppendLine(" AS SemanticSourcePath,") + .AppendLine(" (") + .AppendLine(" SELECT") + .Append(" ").Append(VLatest.VectorSearchParam.ChunkOrdinal, evidenceTableAlias).AppendLine(" AS chunkOrdinal,") + .Append(" ").Append(VLatest.VectorSearchParam.ChunkText, evidenceTableAlias).AppendLine(" AS text,") + .Append(" VECTOR_DISTANCE(").Append(distanceMetricParameter).Append(", ") + .Append(evidenceTableAlias).Append('.').Append(VLatest.VectorSearchParamTable.Embedding).Append(", CAST(") + .Append(queryEmbeddingParameter).Append(" AS VECTOR(").Append(VectorSearchConfiguration.SupportedDimensions).AppendLine("))) AS distance,") + .Append(" ").Append(VLatest.VectorSearchParam.SourceResourceTypeId, evidenceTableAlias).AppendLine(" AS sourceResourceTypeId,") + .Append(" ").Append(VLatest.VectorSearchParam.SourceResourceId, evidenceTableAlias).AppendLine(" AS sourceResourceId,") + .Append(" ").Append(VLatest.VectorSearchParam.SourceResourceVersion, evidenceTableAlias).AppendLine(" AS sourceResourceVersion,") + .Append(" ").Append(VLatest.VectorSearchParam.SourcePath, evidenceTableAlias).Append(" AS sourcePath"); + + if (chainLink != null) + { + StringBuilder + .AppendLine(",") + .Append(" ").Append(VLatest.Resource.ResourceTypeId, witnessTableAlias).AppendLine(" AS witnessResourceTypeId,") + .Append(" ").Append(VLatest.Resource.ResourceId, witnessTableAlias).AppendLine(" AS witnessResourceId,") + .Append(" ").Append(VLatest.Resource.Version, witnessTableAlias).AppendLine(" AS witnessResourceVersion"); + } + else + { + StringBuilder.AppendLine(); + } + + StringBuilder + .Append(" FROM ").Append(VLatest.VectorSearchParam).Append(" AS ").AppendLine(evidenceTableAlias) + .Append(" WHERE ").Append(VLatest.VectorSearchParam.ResourceTypeId, evidenceTableAlias).Append(" = ").Append(VLatest.VectorSearchParam.ResourceTypeId, vectorTableAlias).AppendLine() + .Append(" AND ").Append(VLatest.VectorSearchParam.ResourceSurrogateId, evidenceTableAlias).Append(" = ").Append(VLatest.VectorSearchParam.ResourceSurrogateId, vectorTableAlias).AppendLine() + .Append(" AND ").Append(VLatest.VectorSearchParam.SearchParamId, evidenceTableAlias).Append(" = ").Append(Parameters.AddParameter(VLatest.VectorSearchParam.SearchParamId, searchParamId, includeInHash: true)).AppendLine() + .Append(" AND ").Append(VLatest.VectorSearchParam.EmbeddingModelId, evidenceTableAlias).Append(" = ").Append(Parameters.AddParameter(VLatest.VectorSearchParam.EmbeddingModelId, preparedQuery.EmbeddingModelId, includeInHash: false)).AppendLine() + .Append(" AND VECTOR_DISTANCE(").Append(distanceMetricParameter).Append(", ") + .Append(evidenceTableAlias).Append('.').Append(VLatest.VectorSearchParamTable.Embedding).Append(", CAST(") + .Append(queryEmbeddingParameter).Append(" AS VECTOR(").Append(VectorSearchConfiguration.SupportedDimensions).Append("))) <= ").Append(maximumDistanceParameter).AppendLine() + .Append(" ORDER BY VECTOR_DISTANCE(").Append(distanceMetricParameter).Append(", ") + .Append(evidenceTableAlias).Append('.').Append(VLatest.VectorSearchParamTable.Embedding).Append(", CAST(") + .Append(queryEmbeddingParameter).Append(" AS VECTOR(").Append(VectorSearchConfiguration.SupportedDimensions).Append("))), ") + .Append(VLatest.VectorSearchParam.ChunkOrdinal, evidenceTableAlias).AppendLine(" ASC") + .AppendLine(" FOR JSON PATH") + .AppendLine(" ) AS SemanticEvidenceJson"); + + if (chainLink == null) + { + StringBuilder + .Append(" FROM ").Append(VLatest.VectorSearchParam).Append(" AS ").AppendLine(vectorTableAlias) + .Append(" WHERE ").Append(VLatest.VectorSearchParam.ResourceTypeId, vectorTableAlias).Append(" = ").Append(VLatest.Resource.ResourceTypeId, resourceTableAlias).AppendLine() + .Append(" AND ").Append(VLatest.VectorSearchParam.ResourceSurrogateId, vectorTableAlias).Append(" = ").Append(VLatest.Resource.ResourceSurrogateId, resourceTableAlias).AppendLine() + .Append(" AND "); + } + else + { + short referenceSearchParamId = Model.GetSearchParamId(chainLink.ReferenceSearchParameter.Url); + _searchParamIds.Add(referenceSearchParamId); + StringBuilder.Append(" FROM ").Append(VLatest.ReferenceSearchParam).Append(" AS ").AppendLine(referenceTableAlias); + + if (chainLink.Reversed) + { + StringBuilder + .Append(" JOIN ").Append(VLatest.Resource).Append(" AS ").Append(witnessTableAlias) + .Append(" ON ").Append(VLatest.Resource.ResourceTypeId, witnessTableAlias).Append(" = ").Append(VLatest.ReferenceSearchParam.ResourceTypeId, referenceTableAlias) + .Append(" AND ").Append(VLatest.Resource.ResourceSurrogateId, witnessTableAlias).Append(" = ").AppendLine(VLatest.ReferenceSearchParam.ResourceSurrogateId, referenceTableAlias); + } + else + { + StringBuilder + .Append(" JOIN ").Append(VLatest.Resource).Append(" AS ").Append(witnessTableAlias) + .Append(" ON ").Append(VLatest.Resource.ResourceTypeId, witnessTableAlias).Append(" = ").Append(VLatest.ReferenceSearchParam.ReferenceResourceTypeId, referenceTableAlias) + .Append(" AND ").Append(VLatest.Resource.ResourceId, witnessTableAlias).Append(" = ").AppendLine(VLatest.ReferenceSearchParam.ReferenceResourceId, referenceTableAlias); + } + + StringBuilder + .Append(" JOIN ").Append(VLatest.VectorSearchParam).Append(" AS ").Append(vectorTableAlias) + .Append(" ON ").Append(VLatest.VectorSearchParam.ResourceTypeId, vectorTableAlias).Append(" = ").Append(VLatest.Resource.ResourceTypeId, witnessTableAlias) + .Append(" AND ").Append(VLatest.VectorSearchParam.ResourceSurrogateId, vectorTableAlias).Append(" = ").AppendLine(VLatest.Resource.ResourceSurrogateId, witnessTableAlias) + .Append(" WHERE ").Append(VLatest.ReferenceSearchParam.SearchParamId, referenceTableAlias).Append(" = ").Append(Parameters.AddParameter(VLatest.ReferenceSearchParam.SearchParamId, referenceSearchParamId, includeInHash: true)).AppendLine() + .Append(" AND ").Append(VLatest.ReferenceSearchParam.ResourceTypeId, referenceTableAlias).Append(" IN (") + .Append(string.Join(", ", chainLink.ResourceTypes.Select(resourceType => Parameters.AddParameter(VLatest.ReferenceSearchParam.ResourceTypeId, Model.GetResourceTypeId(resourceType), includeInHash: true)))).AppendLine(")"); + + if (chainLink.Reversed) + { + StringBuilder + .Append(" AND ").Append(VLatest.ReferenceSearchParam.ReferenceResourceTypeId, referenceTableAlias).Append(" = ").Append(VLatest.Resource.ResourceTypeId, resourceTableAlias).AppendLine() + .Append(" AND ").Append(VLatest.ReferenceSearchParam.ReferenceResourceId, referenceTableAlias).Append(" = ").Append(VLatest.Resource.ResourceId, resourceTableAlias).AppendLine(); + } + else + { + StringBuilder + .Append(" AND ").Append(VLatest.ReferenceSearchParam.ResourceTypeId, referenceTableAlias).Append(" = ").Append(VLatest.Resource.ResourceTypeId, resourceTableAlias).AppendLine() + .Append(" AND ").Append(VLatest.ReferenceSearchParam.ResourceSurrogateId, referenceTableAlias).Append(" = ").Append(VLatest.Resource.ResourceSurrogateId, resourceTableAlias).AppendLine(); + } + + IReadOnlyList witnessResourceTypes = chainLink.Reversed + ? chainLink.ResourceTypes + : chainLink.TargetResourceTypes; + StringBuilder + .Append(" AND ").Append(VLatest.Resource.ResourceTypeId, witnessTableAlias).Append(" IN (") + .Append(string.Join(", ", witnessResourceTypes.Select(resourceType => Parameters.AddParameter(VLatest.Resource.ResourceTypeId, Model.GetResourceTypeId(resourceType), includeInHash: true)))).AppendLine(")") + .Append(" AND ").Append(VLatest.Resource.IsHistory, witnessTableAlias).AppendLine(" = 0") + .Append(" AND ").Append(VLatest.Resource.IsDeleted, witnessTableAlias).AppendLine(" = 0") + .Append(" AND "); + } + + StringBuilder + .Append(VLatest.VectorSearchParam.SearchParamId, vectorTableAlias).Append(" = ").Append(Parameters.AddParameter(VLatest.VectorSearchParam.SearchParamId, searchParamId, includeInHash: true)).AppendLine() + .Append(" AND ").Append(VLatest.VectorSearchParam.EmbeddingModelId, vectorTableAlias).Append(" = ").Append(Parameters.AddParameter(VLatest.VectorSearchParam.EmbeddingModelId, preparedQuery.EmbeddingModelId, includeInHash: false)).AppendLine() + .Append(" AND VECTOR_DISTANCE(").Append(distanceMetricParameter).Append(", ") + .Append(vectorTableAlias).Append('.').Append(VLatest.VectorSearchParamTable.Embedding).Append(", CAST(") + .Append(queryEmbeddingParameter).Append(" AS VECTOR(").Append(VectorSearchConfiguration.SupportedDimensions).Append("))) <= ").Append(maximumDistanceParameter).AppendLine() + .Append(" ORDER BY VECTOR_DISTANCE(").Append(distanceMetricParameter).Append(", ") + .Append(vectorTableAlias).Append('.').Append(VLatest.VectorSearchParamTable.Embedding).Append(", CAST(") + .Append(queryEmbeddingParameter).Append(" AS VECTOR(").Append(VectorSearchConfiguration.SupportedDimensions).Append("))), "); + + if (chainLink != null) + { + StringBuilder + .Append(VLatest.VectorSearchParam.ResourceTypeId, vectorTableAlias).Append(" ASC, ") + .Append(VLatest.VectorSearchParam.ResourceSurrogateId, vectorTableAlias).Append(" ASC, "); + } + + StringBuilder + .Append(VLatest.VectorSearchParam.ChunkOrdinal, vectorTableAlias).AppendLine(" ASC") + .AppendLine(" ) semantic"); + } + // TODO: Remove when code starts using TokenSearchParamHighCard table private void AddOptionClause() { @@ -2030,9 +2233,14 @@ private static bool IsPrimaryKeySort(SearchOptions searchOptions) return searchOptions.Sort.All(s => s.searchParameterInfo.Name is SearchParameterNames.ResourceType or SearchParameterNames.LastUpdated); } + private static bool IsScoreSort(SearchOptions searchOptions) + { + return searchOptions.Sort.Count > 0 && searchOptions.Sort[0].searchParameterInfo.Name == SearchParameterNames.Score; + } + internal bool IsSortValueNeeded(SearchOptions context) { - if (context.Sort.Count == 0) + if (context.Sort.Count == 0 || IsScoreSort(context)) { return false; } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/RemoveVectorSearchRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/RemoveVectorSearchRewriter.cs new file mode 100644 index 0000000000..1634eef2a8 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/RemoveVectorSearchRewriter.cs @@ -0,0 +1,70 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors +{ + internal sealed class RemoveVectorSearchRewriter : ExpressionRewriterWithInitialContext + { + public static readonly RemoveVectorSearchRewriter Instance = new RemoveVectorSearchRewriter(); + + public override Expression VisitVectorSearch(VectorSearchExpression expression, object context) + { + return null; + } + + public override Expression VisitChained(ChainedExpression expression, object context) + { + Expression rewrittenExpression = expression.Expression.AcceptVisitor(this, context); + if (ReferenceEquals(rewrittenExpression, expression.Expression)) + { + return expression; + } + + if (rewrittenExpression == null) + { + return null; + } + + return new ChainedExpression( + expression.ResourceTypes, + expression.ReferenceSearchParameter, + expression.TargetResourceTypes, + expression.Reversed, + rewrittenExpression); + } + + public override Expression VisitMultiary(MultiaryExpression expression, object context) + { + List rewrittenExpressions = null; + + for (int index = 0; index < expression.Expressions.Count; index++) + { + Expression originalExpression = expression.Expressions[index]; + Expression rewrittenExpression = originalExpression.AcceptVisitor(this, context); + + if (!ReferenceEquals(originalExpression, rewrittenExpression)) + { + EnsureAllocatedAndPopulated(ref rewrittenExpressions, expression.Expressions, index); + } + + if (rewrittenExpression != null) + { + rewrittenExpressions?.Add(rewrittenExpression); + } + } + + return rewrittenExpressions switch + { + null => expression, + { Count: 0 } => null, + { Count: 1 } => rewrittenExpressions[0], + _ => new MultiaryExpression(expression.MultiaryOperation, rewrittenExpressions), + }; + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/SearchParamTableExpressionQueryGeneratorFactory.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/SearchParamTableExpressionQueryGeneratorFactory.cs index 0dbafe1b7f..b8aca03ae8 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/SearchParamTableExpressionQueryGeneratorFactory.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/SearchParamTableExpressionQueryGeneratorFactory.cs @@ -36,6 +36,11 @@ public SearchParamTableExpressionQueryGenerator VisitSearchParameter(SearchParam return VisitSearchParameterExpressionBase(expression.Parameter, expression.Expression, context); } + public SearchParamTableExpressionQueryGenerator VisitVectorSearch(VectorSearchExpression expression, object context) + { + throw new InvalidOperationException("Vector search expressions require the vector query execution path."); + } + public SearchParamTableExpressionQueryGenerator VisitMissingSearchParameter(MissingSearchParameterExpression expression, object context) { return VisitSearchParameterExpressionBase(expression.Parameter, null, context); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/SortRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/SortRewriter.cs index 658f49c84e..1b1cd57420 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/SortRewriter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/SortRewriter.cs @@ -36,6 +36,11 @@ public override Expression VisitSqlRoot(SqlRootExpression expression, SqlSearchO return expression; } + if (context.Sort[0].searchParameterInfo.Name == SearchParameterNames.Score) + { + return expression; + } + // _type and _lastUpdated sort params are handled differently than others, because they can be // inferred directly from the resource table itself. if (context.Sort.All(s => s.searchParameterInfo.Name is SearchParameterNames.ResourceType or SearchParameterNames.LastUpdated)) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/TopRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/TopRewriter.cs index 7212adc790..a3a4341aa9 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/TopRewriter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/TopRewriter.cs @@ -17,7 +17,9 @@ internal class TopRewriter : SqlExpressionRewriter public override Expression VisitSqlRoot(SqlRootExpression expression, SearchOptions context) { - if (context.CountOnly || expression.SearchParamTableExpressions.Count == 0) + if (context.CountOnly || + expression.SearchParamTableExpressions.Count == 0 || + context is SqlSearchOptions { PreparedVectorQuery: not null }) { return expression; } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlDocumentReferenceSemanticSearch.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlDocumentReferenceSemanticSearch.cs new file mode 100644 index 0000000000..56f9c29de5 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlDocumentReferenceSemanticSearch.cs @@ -0,0 +1,145 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.SqlServer.Features.Storage; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Search.SemanticSearch +{ + /// + /// Embeds a query and ranks candidate DocumentReference vectors in SQL Server. + /// + public sealed class SqlDocumentReferenceSemanticSearch : IDocumentReferenceSemanticSearch + { + private readonly IEmbeddingClient _embeddingClient; + private readonly IVectorStore _vectorStore; + private readonly IEmbeddingModelRegistry _embeddingModelRegistry; + private readonly IVectorSearchParameterResolver _searchParameterResolver; + private readonly SqlServerFhirModel _model; + private readonly VectorSearchConfiguration _configuration; + + /// + /// Initializes a new instance of the class. + /// + public SqlDocumentReferenceSemanticSearch( + IEmbeddingClient embeddingClient, + IVectorStore vectorStore, + IEmbeddingModelRegistry embeddingModelRegistry, + IVectorSearchParameterResolver searchParameterResolver, + SqlServerFhirModel model, + IOptions configuration) + { + _embeddingClient = EnsureArg.IsNotNull(embeddingClient, nameof(embeddingClient)); + _vectorStore = EnsureArg.IsNotNull(vectorStore, nameof(vectorStore)); + _embeddingModelRegistry = EnsureArg.IsNotNull(embeddingModelRegistry, nameof(embeddingModelRegistry)); + _searchParameterResolver = EnsureArg.IsNotNull(searchParameterResolver, nameof(searchParameterResolver)); + _model = EnsureArg.IsNotNull(model, nameof(model)); + _configuration = EnsureArg.IsNotNull(configuration, nameof(configuration)).Value; + } + + /// + public async Task> SearchAsync( + string query, + IReadOnlyList candidates, + int count, + CancellationToken cancellationToken) + { + EnsureArg.IsNotNullOrWhiteSpace(query, nameof(query)); + EnsureArg.IsNotNull(candidates, nameof(candidates)); + + if (candidates.Count == 0) + { + return System.Array.Empty(); + } + + IReadOnlyList embeddings = await _embeddingClient.GenerateEmbeddingsAsync(new[] { query }, cancellationToken); + short embeddingModelId = await _embeddingModelRegistry.GetEmbeddingModelIdAsync(cancellationToken); + var rankedHits = new List<(string ResourceType, ResourceWrapper Owner, SearchParameterInfo SearchParameter, VectorSearchHit Hit)>(); + + foreach (IGrouping candidatesByResourceType in candidates.GroupBy(candidate => candidate.ResourceTypeName, System.StringComparer.Ordinal)) + { + string resourceType = candidatesByResourceType.Key; + IReadOnlyList searchParameters = _searchParameterResolver.GetSearchParameters(resourceType); + IReadOnlyList candidateIds = candidatesByResourceType.Select(candidate => candidate.ResourceSurrogateId).ToList(); + Dictionary candidatesBySurrogateId = candidatesByResourceType.ToDictionary(candidate => candidate.ResourceSurrogateId); + + foreach (SearchParameterInfo searchParameter in searchParameters) + { + IReadOnlyList targetHits = await _vectorStore.SearchAsync( + _model.GetResourceTypeId(resourceType), + _model.GetSearchParamId(searchParameter.Url), + embeddingModelId, + _configuration.Query.DistanceMetric, + embeddings[0], + candidateIds, + count, + _configuration.Query.EvidenceCount, + cancellationToken); + + foreach (VectorSearchHit hit in targetHits) + { + ResourceWrapper owner = candidatesBySurrogateId[hit.ResourceSurrogateId]; + rankedHits.Add((resourceType, owner, searchParameter, hit)); + } + } + } + + return rankedHits + .GroupBy(result => (result.ResourceType, result.Hit.ResourceSurrogateId)) + .Select(group => + { + var orderedHits = group + .OrderByDescending(result => result.Hit.Score) + .ThenBy(result => result.Hit.ChunkOrdinal) + .ThenBy(result => result.SearchParameter.Url.AbsoluteUri, System.StringComparer.Ordinal) + .Take(_configuration.Query.EvidenceCount) + .ToList(); + IReadOnlyList evidenceItems = orderedHits + .Select(result => CreateEvidence(result.Owner, result.SearchParameter, result.Hit)) + .ToList(); + + return new VectorSearchResult( + group.Key.ResourceType, + group.Key.ResourceSurrogateId, + orderedHits[0].Hit.Score, + evidenceItems); + }) + .OrderByDescending(result => result.Score) + .Take(count) + .ToList(); + } + + private SemanticSearchEvidence CreateEvidence( + ResourceWrapper owner, + SearchParameterInfo searchParameter, + VectorSearchHit hit) + { + string sourceResourceType = hit.SourceResourceTypeId.HasValue + ? _model.GetResourceTypeName(hit.SourceResourceTypeId.Value) + : owner.ResourceTypeName; + var sourceKey = new ResourceKey( + sourceResourceType, + hit.SourceResourceId ?? owner.ResourceId, + hit.SourceResourceVersion ?? owner.Version); + + return new SemanticSearchEvidence( + hit.ChunkText, + hit.ChunkOrdinal, + (decimal)hit.Score, + searchParameter.Url, + sourceKey.ToString(), + hit.SourcePath ?? searchParameter.Expression); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlEmbeddingModelRegistry.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlEmbeddingModelRegistry.cs new file mode 100644 index 0000000000..7a462eaed6 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlEmbeddingModelRegistry.cs @@ -0,0 +1,115 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Data; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Search.SemanticSearch +{ + /// + /// Resolves and caches the database identifier for the configured embedding model. + /// + public sealed class SqlEmbeddingModelRegistry : IEmbeddingModelRegistry, IDisposable + { + private const string ResolveEmbeddingModel = @" +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; + +DECLARE @EmbeddingModelId SMALLINT; + +SELECT @EmbeddingModelId = EmbeddingModelId +FROM dbo.EmbeddingModel WITH (UPDLOCK, HOLDLOCK) +WHERE ModelName = @ModelName + AND ModelVersion = @ModelVersion; + +IF @EmbeddingModelId IS NULL +BEGIN + INSERT INTO dbo.EmbeddingModel (ModelName, ModelVersion, Dimension, DistanceMetric) + VALUES (@ModelName, @ModelVersion, @Dimension, @DistanceMetric); + + SET @EmbeddingModelId = CONVERT(SMALLINT, SCOPE_IDENTITY()); +END +ELSE IF EXISTS +( + SELECT 1 + FROM dbo.EmbeddingModel + WHERE EmbeddingModelId = @EmbeddingModelId + AND (Dimension <> @Dimension OR DistanceMetric <> @DistanceMetric) +) +BEGIN + THROW 50000, 'The configured embedding model metadata does not match the existing registry row.', 1; +END; + +COMMIT TRANSACTION; +SELECT @EmbeddingModelId;"; + + private readonly string _connectionString; + private readonly VectorSearchConfiguration _configuration; + private readonly SemaphoreSlim _initializationLock = new SemaphoreSlim(1, 1); + private short? _embeddingModelId; + + /// + /// Initializes a new instance of the class. + /// + /// The SQL database connection string. + /// The vector-search configuration. + public SqlEmbeddingModelRegistry(string connectionString, IOptions configuration) + { + _connectionString = EnsureArg.IsNotNullOrWhiteSpace(connectionString, nameof(connectionString)); + _configuration = EnsureArg.IsNotNull(configuration, nameof(configuration)).Value; + } + + /// + public async Task GetEmbeddingModelIdAsync(CancellationToken cancellationToken) + { + if (_embeddingModelId.HasValue) + { + return _embeddingModelId.Value; + } + + await _initializationLock.WaitAsync(cancellationToken); + try + { + if (_embeddingModelId.HasValue) + { + return _embeddingModelId.Value; + } + + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(cancellationToken); + + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = ResolveEmbeddingModel; + command.Parameters.Add("@ModelName", SqlDbType.VarChar, 128).Value = _configuration.Embedding.ModelName; + command.Parameters.Add("@ModelVersion", SqlDbType.VarChar, 64).Value = _configuration.Embedding.ModelVersion; + command.Parameters.Add("@Dimension", SqlDbType.Int).Value = _configuration.Embedding.Dimensions; + command.Parameters.Add("@DistanceMetric", SqlDbType.VarChar, 16).Value = _configuration.Query.DistanceMetric; + + object result = await command.ExecuteScalarAsync(cancellationToken); + _embeddingModelId = Convert.ToInt16(result, CultureInfo.InvariantCulture); + return _embeddingModelId.Value; + } + finally + { + _initializationLock.Release(); + } + } + + /// + public void Dispose() + { + _initializationLock.Dispose(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlVectorFormatter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlVectorFormatter.cs new file mode 100644 index 0000000000..1be350c7a1 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlVectorFormatter.cs @@ -0,0 +1,36 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using EnsureThat; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Search.SemanticSearch +{ + internal static class SqlVectorFormatter + { + public static string Format(IReadOnlyList embedding) + { + EnsureArg.IsNotNull(embedding, nameof(embedding)); + + var builder = new StringBuilder((embedding.Count * 8) + 2); + builder.Append('['); + + for (int index = 0; index < embedding.Count; index++) + { + if (index > 0) + { + builder.Append(','); + } + + builder.Append(embedding[index].ToString("R", CultureInfo.InvariantCulture)); + } + + builder.Append(']'); + return builder.ToString(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlVectorResourceReader.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlVectorResourceReader.cs new file mode 100644 index 0000000000..b101f0804b --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlVectorResourceReader.cs @@ -0,0 +1,53 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.SqlServer.Features.Storage; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Search.SemanticSearch +{ + /// + /// Reads persisted resources needed to resolve vector source text without depending on the FHIR data store. + /// + internal sealed class SqlVectorResourceReader : IVectorResourceReader + { + private readonly SqlStoreClient _storeClient; + private readonly SqlServerFhirModel _model; + private readonly ICompressedRawResourceConverter _compressedRawResourceConverter; + + public SqlVectorResourceReader( + SqlStoreClient storeClient, + SqlServerFhirModel model, + ICompressedRawResourceConverter compressedRawResourceConverter) + { + _storeClient = EnsureArg.IsNotNull(storeClient, nameof(storeClient)); + _model = EnsureArg.IsNotNull(model, nameof(model)); + _compressedRawResourceConverter = EnsureArg.IsNotNull(compressedRawResourceConverter, nameof(compressedRawResourceConverter)); + } + + /// + public async Task GetAsync(ResourceKey key, CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(key, nameof(key)); + + IReadOnlyList resources = await _storeClient.GetAsync( + new[] { key }, + _model.GetResourceTypeId, + _compressedRawResourceConverter.ReadCompressedRawResource, + _model.GetResourceTypeName, + isReadOnly: true, + cancellationToken); + + return resources.SingleOrDefault(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlVectorStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlVectorStore.cs new file mode 100644 index 0000000000..56c1107b98 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SemanticSearch/SqlVectorStore.cs @@ -0,0 +1,241 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Data; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Data.SqlClient; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Search.SemanticSearch +{ + /// + /// A prototype that writes passage vectors to the VectorSearchParam table + /// with a direct parameterized insert, passing each vector as a JSON array cast to the SQL VECTOR type. + /// The production path writes these rows through the merge pipeline (see the design doc, section 11). + /// + public sealed class SqlVectorStore : IVectorStore + { + private const string InsertChunk = @" +INSERT INTO dbo.VectorSearchParam + (ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, Embedding) +VALUES + (@ResourceTypeId, @ResourceSurrogateId, @SearchParamId, @ChunkOrdinal, @EmbeddingModelId, @ChunkText, @SourceTextHash, CAST(@Embedding AS VECTOR(1536)));"; + + private const string DeleteResourceChunks = @" +DELETE FROM dbo.VectorSearchParam +WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId = @ResourceSurrogateId + AND SearchParamId = @SearchParamId;"; + + // The candidate ids arrive as one comma-delimited bound parameter and are split server-side, so the query text + // stays a compile-time constant (no user input is ever concatenated into it) while every value is parameterized. + private const string SearchChunks = @" +WITH RankedChunks AS +( + SELECT v.ResourceSurrogateId, + v.ChunkOrdinal, + v.ChunkText, + v.SourceResourceTypeId, + v.SourceResourceId, + v.SourceResourceVersion, + v.SourcePath, + VECTOR_DISTANCE(@DistanceMetric, v.Embedding, CAST(@QueryEmbedding AS VECTOR(1536))) AS Distance, + ROW_NUMBER() OVER + ( + PARTITION BY v.ResourceSurrogateId + ORDER BY VECTOR_DISTANCE(@DistanceMetric, v.Embedding, CAST(@QueryEmbedding AS VECTOR(1536))), v.ChunkOrdinal + ) AS ChunkRank + FROM dbo.VectorSearchParam AS v + WHERE v.ResourceTypeId = @ResourceTypeId + AND v.SearchParamId = @SearchParamId + AND v.EmbeddingModelId = @EmbeddingModelId + AND v.ResourceSurrogateId IN (SELECT CAST(value AS BIGINT) FROM STRING_SPLIT(@CandidateIds, ',')) +), +BestResources AS +( + SELECT ResourceSurrogateId, + MIN(Distance) AS BestDistance + FROM RankedChunks + GROUP BY ResourceSurrogateId +), +SelectedResources AS +( + SELECT TOP (@MaxResults) + ResourceSurrogateId, + BestDistance + FROM BestResources + ORDER BY BestDistance, ResourceSurrogateId +) +SELECT c.ResourceSurrogateId, + c.ChunkOrdinal, + c.ChunkText, + c.SourceResourceTypeId, + c.SourceResourceId, + c.SourceResourceVersion, + c.SourcePath, + c.Distance +FROM RankedChunks AS c +INNER JOIN SelectedResources AS s + ON s.ResourceSurrogateId = c.ResourceSurrogateId +WHERE c.ChunkRank <= @EvidenceCount +ORDER BY s.BestDistance, s.ResourceSurrogateId, c.ChunkRank;"; + + private readonly string _connectionString; + + /// + /// Initializes a new instance of the class. + /// + /// The connection string for the FHIR SQL database. + public SqlVectorStore(string connectionString) + { + EnsureArg.IsNotNullOrWhiteSpace(connectionString, nameof(connectionString)); + + _connectionString = connectionString; + } + + /// + public async Task StoreAsync( + short resourceTypeId, + long resourceSurrogateId, + short searchParamId, + short embeddingModelId, + IReadOnlyList chunks, + CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(chunks, nameof(chunks)); + + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(cancellationToken); + + await using SqlTransaction transaction = (SqlTransaction)await connection.BeginTransactionAsync(cancellationToken); + await using (SqlCommand deleteCommand = connection.CreateCommand()) + { + deleteCommand.Transaction = transaction; + deleteCommand.CommandText = DeleteResourceChunks; + AddResourceParameters(deleteCommand, resourceTypeId, resourceSurrogateId, searchParamId, embeddingModelId); + await deleteCommand.ExecuteNonQueryAsync(cancellationToken); + } + + foreach (VectorSearchChunk chunk in chunks) + { + await using SqlCommand command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = InsertChunk; + + AddResourceParameters(command, resourceTypeId, resourceSurrogateId, searchParamId, embeddingModelId); + command.Parameters.Add("@ChunkOrdinal", SqlDbType.SmallInt).Value = chunk.ChunkOrdinal; + command.Parameters.Add("@ChunkText", SqlDbType.NVarChar, -1).Value = chunk.ChunkText; + + byte[] hash = ToArray(chunk.SourceTextHash); + command.Parameters.Add("@SourceTextHash", SqlDbType.Binary, hash.Length).Value = hash; + command.Parameters.Add("@Embedding", SqlDbType.NVarChar, -1).Value = SqlVectorFormatter.Format(chunk.Embedding); + + await command.ExecuteNonQueryAsync(cancellationToken); + } + + await transaction.CommitAsync(cancellationToken); + } + + /// + public async Task> SearchAsync( + short resourceTypeId, + short searchParamId, + short embeddingModelId, + string distanceMetric, + IReadOnlyList queryEmbedding, + IReadOnlyList candidateResourceSurrogateIds, + int maxResults, + int evidenceCount, + CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(queryEmbedding, nameof(queryEmbedding)); + EnsureArg.IsNotNull(candidateResourceSurrogateIds, nameof(candidateResourceSurrogateIds)); + EnsureArg.IsNotNullOrWhiteSpace(distanceMetric, nameof(distanceMetric)); + EnsureArg.IsGt(maxResults, 0, nameof(maxResults)); + EnsureArg.IsGt(evidenceCount, 0, nameof(evidenceCount)); + + // Nothing passed the structured filter, so there is nothing to rank. + if (candidateResourceSurrogateIds.Count == 0) + { + return Array.Empty(); + } + + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(cancellationToken); + + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = SearchChunks; + + command.Parameters.Add("@MaxResults", SqlDbType.Int).Value = maxResults; + command.Parameters.Add("@EvidenceCount", SqlDbType.Int).Value = evidenceCount; + command.Parameters.Add("@ResourceTypeId", SqlDbType.SmallInt).Value = resourceTypeId; + command.Parameters.Add("@SearchParamId", SqlDbType.SmallInt).Value = searchParamId; + command.Parameters.Add("@EmbeddingModelId", SqlDbType.SmallInt).Value = embeddingModelId; + command.Parameters.Add("@DistanceMetric", SqlDbType.VarChar, 16).Value = distanceMetric; + command.Parameters.Add("@QueryEmbedding", SqlDbType.NVarChar, -1).Value = SqlVectorFormatter.Format(queryEmbedding); + command.Parameters.Add("@CandidateIds", SqlDbType.NVarChar, -1).Value = string.Join(",", candidateResourceSurrogateIds); + + var results = new List(); + + await using SqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + long resourceSurrogateId = reader.GetInt64(0); + int chunkOrdinal = reader.GetInt16(1); + string chunkText = reader.GetString(2); + short? sourceResourceTypeId = await reader.IsDBNullAsync(3, cancellationToken) ? null : reader.GetInt16(3); + string sourceResourceId = await reader.IsDBNullAsync(4, cancellationToken) ? null : reader.GetString(4); + string sourceResourceVersion = await reader.IsDBNullAsync(5, cancellationToken) ? null : reader.GetString(5); + string sourcePath = await reader.IsDBNullAsync(6, cancellationToken) ? null : reader.GetString(6); + double distance = Convert.ToDouble(reader.GetValue(7), CultureInfo.InvariantCulture); + + // The supported cosine distance is 0 (identical) to 2 (opposite); map it to a 0..1 relevance score where higher is better. + float score = (float)Math.Clamp(1.0 - (distance / 2.0), 0.0, 1.0); + + results.Add(new VectorSearchHit( + resourceSurrogateId, + chunkOrdinal, + chunkText, + score, + sourceResourceTypeId, + sourceResourceId, + sourceResourceVersion, + sourcePath)); + } + + return results; + } + + private static void AddResourceParameters( + SqlCommand command, + short resourceTypeId, + long resourceSurrogateId, + short searchParamId, + short embeddingModelId) + { + command.Parameters.Add("@ResourceTypeId", SqlDbType.SmallInt).Value = resourceTypeId; + command.Parameters.Add("@ResourceSurrogateId", SqlDbType.BigInt).Value = resourceSurrogateId; + command.Parameters.Add("@SearchParamId", SqlDbType.SmallInt).Value = searchParamId; + command.Parameters.Add("@EmbeddingModelId", SqlDbType.SmallInt).Value = embeddingModelId; + } + + private static byte[] ToArray(IReadOnlyList source) + { + var result = new byte[source.Count]; + + for (int i = 0; i < result.Length; i++) + { + result[i] = source[i]; + } + + return result; + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs index 5ea3620684..2ac318726d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; namespace Microsoft.Health.Fhir.SqlServer.Features.Search { @@ -35,6 +36,17 @@ public SqlSearchOptions(SearchOptions searchOptions) /// public bool SortHasMissingModifier { get; internal set; } + /// + /// Gets or sets the prepared vector query used to rank structured matches before pagination. + /// + public PreparedVectorSearchQuery PreparedVectorQuery { get; set; } + + internal double? SemanticContinuationDistance { get; set; } + + internal short? SemanticContinuationResourceTypeId { get; set; } + + internal long? SemanticContinuationResourceSurrogateId { get; set; } + /// /// Set when a SMART compartment membership context was attached to the root expression for this /// search. The SQL query generator re-checks this flag so that a rewrite step that reconstructs the diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchParameterValidator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchParameterValidator.cs index 4e0cc3c8ac..3dfd953f83 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchParameterValidator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchParameterValidator.cs @@ -8,6 +8,7 @@ using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors; using Microsoft.Health.Fhir.SqlServer.Features.Storage; +using Microsoft.Health.Fhir.ValueSets; namespace Microsoft.Health.Fhir.SqlServer.Features.Search { @@ -27,6 +28,17 @@ public bool ValidateSearchParameter(SearchParameterInfo searchParameter, out str EnsureArg.IsNotNull(searchParameter, nameof(searchParameter)); errorMessage = null; + if (searchParameter.VectorConfig != null) + { + if (searchParameter.Type == SearchParamType.Special) + { + return true; + } + + errorMessage = string.Format(Resources.SearchParameterTypeNotSupportedBySQLServer, searchParameter.Type); + return false; + } + var factory = new SearchParamTableExpressionQueryGeneratorFactory(_searchParameterToSearchValueTypeMap); try diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index fcabe5ab2b..abf76e8002 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -14,6 +14,7 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -31,6 +32,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.SqlServer.Features.Schema; using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; @@ -69,6 +71,14 @@ internal class SqlServerSearchService : SearchService /// internal const string ReferenceResourceTypeFilteredStatsParameterId = "Search.ReferenceResourceTypeFilteredStats.IsEnabled"; private const string SortValueColumnName = "SortValue"; + private const string SemanticDistanceColumnName = "SemanticDistance"; + private const string SemanticChunkOrdinalColumnName = "SemanticChunkOrdinal"; + private const string SemanticChunkTextColumnName = "SemanticChunkText"; + private const string SemanticSourceResourceTypeIdColumnName = "SemanticSourceResourceTypeId"; + private const string SemanticSourceResourceIdColumnName = "SemanticSourceResourceId"; + private const string SemanticSourceResourceVersionColumnName = "SemanticSourceResourceVersion"; + private const string SemanticSourcePathColumnName = "SemanticSourcePath"; + private const string SemanticEvidenceJsonColumnName = "SemanticEvidenceJson"; private readonly ISqlServerFhirModel _model; private readonly SqlRootExpressionRewriter _sqlRootExpressionRewriter; @@ -91,6 +101,7 @@ internal class SqlServerSearchService : SearchService private readonly ISqlQueryHashCalculator _queryHashCalculator; private readonly IFhirDataStore _fhirDataStore; private readonly IQueryPlanReuseChecker _queryPlanReuseChecker; + private readonly IVectorSearchQueryProcessor _vectorSearchQueryProcessor; private static readonly string[] NewLineSeparators = ["\r\n", "\n"]; private static readonly Regex WhitespacePattern = new Regex(@"\s+", RegexOptions.Compiled); @@ -163,7 +174,8 @@ public SqlServerSearchService( ICompressedRawResourceConverter compressedRawResourceConverter, ISqlQueryHashCalculator queryHashCalculator, IQueryPlanReuseChecker queryPlanReuseChecker, - ILogger logger) + ILogger logger, + IVectorSearchQueryProcessor vectorSearchQueryProcessor = null) : base(searchOptionsFactory, fhirDataStore, logger) { EnsureArg.IsNotNull(sqlRootExpressionRewriter, nameof(sqlRootExpressionRewriter)); @@ -192,6 +204,7 @@ public SqlServerSearchService( _sqlRetryService = sqlRetryService; _queryHashCalculator = queryHashCalculator; _queryPlanReuseChecker = queryPlanReuseChecker; + _vectorSearchQueryProcessor = vectorSearchQueryProcessor; _logger = logger; _schemaInformation = schemaInformation; @@ -231,6 +244,17 @@ private static void InitializeProcessingFlags(ILogger lo public override async Task SearchAsync(SearchOptions searchOptions, CancellationToken cancellationToken) { SqlSearchOptions sqlSearchOptions = new SqlSearchOptions(searchOptions); + if (_vectorSearchQueryProcessor != null) + { + sqlSearchOptions.PreparedVectorQuery = await _vectorSearchQueryProcessor.PrepareAsync(sqlSearchOptions.Expression, cancellationToken); + } + + if (sqlSearchOptions.PreparedVectorQuery?.SearchParameter.VectorConfig?.SourceStrategy == VectorTextSourceStrategy.LocalBinaryReference && + (sqlSearchOptions.CountOnly || sqlSearchOptions.IncludeTotal == TotalType.Accurate)) + { + throw new InvalidSearchOperationException( + "Exact totals are not supported for semantic searches whose SearchParameter uses the localBinaryReference source strategy."); + } if (sqlSearchOptions.IsIncludesOperation) { @@ -318,7 +342,8 @@ public override async Task SearchAsync(SearchOptions searchOptions resultCount <= sqlSearchOptions.MaxItemCount && sqlSearchOptions.Sort != null && sqlSearchOptions.Sort.Count > 0 && - sqlSearchOptions.Sort[0].searchParameterInfo.Code != KnownQueryParameterNames.LastUpdated) + sqlSearchOptions.Sort[0].searchParameterInfo.Code != KnownQueryParameterNames.LastUpdated && + !IsScoreSort(sqlSearchOptions)) { // We seem to have run a sort which has returned less results than what max we can return. // Let's determine whether we need to execute another query or not. @@ -482,6 +507,11 @@ private async Task RunSearch(SqlSearchOptions sqlSearchOptions, Ca } } + private static bool ContainsVectorSearch(Expression expression) + { + return expression?.AcceptVisitor(VectorSearchPresenceVisitor.Instance, context: null) ?? false; + } + private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, bool reuseQueryPlans, CancellationToken cancellationToken) { if (sqlSearchOptions.IsIncludesOperation) @@ -490,7 +520,9 @@ private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, b } Stopwatch stopwatch = Stopwatch.StartNew(); - Expression searchExpression = sqlSearchOptions.Expression; + Expression searchExpression = sqlSearchOptions.PreparedVectorQuery == null + ? sqlSearchOptions.Expression + : sqlSearchOptions.Expression?.AcceptVisitor(RemoveVectorSearchRewriter.Instance); // AND in the continuation token if (!string.IsNullOrWhiteSpace(sqlSearchOptions.ContinuationToken) && !sqlSearchOptions.CountOnly) @@ -498,7 +530,19 @@ private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, b var continuationToken = ContinuationToken.FromString(sqlSearchOptions.ContinuationToken); if (continuationToken != null) { - if (string.IsNullOrEmpty(continuationToken.SortValue)) + if (IsRelevanceSort(sqlSearchOptions)) + { + if (!continuationToken.TryGetSemanticCursor(out double distance, out short resourceTypeId, out long resourceSurrogateId)) + { + _logger.LogWarning("Bad Request (InvalidContinuationToken)"); + throw new BadRequestException(Resources.InvalidContinuationToken); + } + + sqlSearchOptions.SemanticContinuationDistance = distance; + sqlSearchOptions.SemanticContinuationResourceTypeId = resourceTypeId; + sqlSearchOptions.SemanticContinuationResourceSurrogateId = resourceSurrogateId; + } + else if (string.IsNullOrEmpty(continuationToken.SortValue)) { // Check whether it's a _lastUpdated or (_type,_lastUpdated) sort optimization bool optimize = true; @@ -565,7 +609,8 @@ private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, b // Reads by resource ids is handled directly via GetAsync(). // Search result is set only on success, otherwise it is null. // SqlServerFhirDataStore uses the same retry class, so it is not needed to call this inside _sqlRetryService.ExecuteSql down below. - if (await GetResourcesByIdsAsync(expression, clonedSearchOptions, _fhirDataStore, cancellationToken) is SearchResult result) + if (clonedSearchOptions.PreparedVectorQuery == null && + await GetResourcesByIdsAsync(expression, clonedSearchOptions, _fhirDataStore, cancellationToken) is SearchResult result) { _logger.LogInformation("Get resources by ids was handled via GetAsync()"); return result; @@ -586,7 +631,8 @@ await _sqlRetryService.ExecuteSql( PopulateSqlCommandFromQueryHints(clonedSearchOptions, sqlCommand); sqlCommand.CommandTimeout = 1200; // set to 20 minutes, as dataset is usually large } - else if (TryExtractGetResourcesByTokensParams(expression, clonedSearchOptions, (SqlServerFhirModel)_model, out var resourceTypeId, out var searchParamId, out var tokens, out var top)) + else if (clonedSearchOptions.PreparedVectorQuery == null && + TryExtractGetResourcesByTokensParams(expression, clonedSearchOptions, (SqlServerFhirModel)_model, out var resourceTypeId, out var searchParamId, out var tokens, out var top)) { PopulateGetResourcesByTokensCommand(sqlCommand, resourceTypeId, searchParamId, tokens, top); } @@ -680,6 +726,7 @@ await _sqlRetryService.ExecuteSql( ReadWrapper( reader, exportTimeTravel, + sqlSearchOptions.PreparedVectorQuery != null, out short resourceTypeId, out string resourceId, out int version, @@ -692,7 +739,15 @@ await _sqlRetryService.ExecuteSql( out string searchParameterHash, out byte[] rawResourceBytes, out bool isInvisible, - out bool isHistory); + out bool isHistory, + out double? semanticDistance, + out int? semanticChunkOrdinal, + out string semanticChunkText, + out short? semanticSourceResourceTypeId, + out string semanticSourceResourceId, + out string semanticSourceResourceVersion, + out string semanticSourcePath, + out string semanticEvidenceJson); if (isInvisible) { @@ -741,15 +796,18 @@ await _sqlRetryService.ExecuteSql( // If sort value needed, that means we have an extra column tracking sort value. // Keep track of sort value if this is the last row. - if (matchCount == clonedSearchOptions.MaxItemCount - 1 && isSortValueNeeded) + if (matchCount == clonedSearchOptions.MaxItemCount - 1 && IsScoreSort(clonedSearchOptions) && semanticDistance.HasValue) + { + sortValue = semanticDistance.Value.ToString("R", CultureInfo.InvariantCulture); + } + else if (matchCount == clonedSearchOptions.MaxItemCount - 1 && isSortValueNeeded) { var tempSortValue = reader.GetValue(SortValueColumnName); sortValue = (tempSortValue as DateTime?) != null ? (tempSortValue as DateTime?).Value.ToString("o") : tempSortValue.ToString(); } matchCount++; - matchedResources.Add(new SearchResultEntry( - new ResourceWrapper( + var resourceWrapper = new ResourceWrapper( resourceId, version.ToString(CultureInfo.InvariantCulture), _model.GetResourceTypeName(resourceTypeId), @@ -764,8 +822,40 @@ await _sqlRetryService.ExecuteSql( resourceSurrogateId) { IsHistory = isHistory, - }, - SearchEntryMode.Match)); + }; + + SemanticSearchEvidence semanticEvidence = null; + IReadOnlyList semanticEvidenceItems = Array.Empty(); + decimal? semanticScore = null; + if (semanticDistance.HasValue) + { + semanticScore = NormalizeCosineDistance(semanticDistance.Value); + semanticEvidenceItems = DeserializeSemanticEvidence( + semanticEvidenceJson, + sqlSearchOptions.PreparedVectorQuery, + resourceWrapper); + + if (semanticEvidenceItems.Count == 0 && sqlSearchOptions.PreparedVectorQuery.ChainLinks.Count == 0) + { + semanticEvidence = new SemanticSearchEvidence( + semanticChunkText, + semanticChunkOrdinal.Value, + semanticScore, + sqlSearchOptions.PreparedVectorQuery.SearchParameter.Url, + new ResourceKey( + semanticSourceResourceTypeId.HasValue ? _model.GetResourceTypeName(semanticSourceResourceTypeId.Value) : resourceWrapper.ResourceTypeName, + semanticSourceResourceId ?? resourceWrapper.ResourceId, + semanticSourceResourceVersion ?? resourceWrapper.Version).ToString(), + semanticSourcePath ?? sqlSearchOptions.PreparedVectorQuery.SearchParameter.Expression); + } + } + + matchedResources.Add(new SearchResultEntry( + resourceWrapper, + SearchEntryMode.Match, + semanticScore, + semanticEvidence, + semanticEvidenceItems)); } else { @@ -853,7 +943,8 @@ await _sqlRetryService.ExecuteSql( // If this is a sort query, lets keep track of whether we actually searched for sort values. if (clonedSearchOptions.Sort != null && clonedSearchOptions.Sort.Count > 0 && - clonedSearchOptions.Sort[0].searchParameterInfo.Code != KnownQueryParameterNames.LastUpdated) + clonedSearchOptions.Sort[0].searchParameterInfo.Code != KnownQueryParameterNames.LastUpdated && + !IsScoreSort(clonedSearchOptions)) { // If there is an extra column for sort value, we know we have searched for sort values. If no results were returned, we don't know if we have searched for sort values so we need to assume we did so we run the second phase. sqlSearchOptions.DidWeSearchForSortValue = isSortValueNeeded; @@ -866,6 +957,11 @@ await _sqlRetryService.ExecuteSql( sqlSearchOptions.IsSortWithFilter = true; } + if (sqlSearchOptions.PreparedVectorQuery != null) + { + AssignEvidenceRanks(matchedResources); + } + if (clonedSearchOptions.SortHasMissingModifier) { sqlSearchOptions.SortHasMissingModifier = true; @@ -981,6 +1077,7 @@ await _sqlRetryService.ExecuteSql( ReadWrapper( reader, true, + false, out short _, out string resourceId, out int version, @@ -993,7 +1090,15 @@ await _sqlRetryService.ExecuteSql( out string searchParameterHash, out byte[] rawResourceBytes, out bool isInvisible, - out bool isHistory); + out bool isHistory, + out double? _, + out int? _, + out string _, + out short? _, + out string _, + out string _, + out string _, + out string _); if (isInvisible) { @@ -1727,9 +1832,22 @@ public override async Task> GetUsedResourceTypes(Cancellat /// The input SearchOptions /// The searchExpression /// If the sort needs to be updated, a new instance, otherwise, the same instance as - private SqlSearchOptions UpdateSort(SqlSearchOptions searchOptions, Expression searchExpression) + internal SqlSearchOptions UpdateSort(SqlSearchOptions searchOptions, Expression searchExpression) { SqlSearchOptions newSearchOptions = searchOptions; + if (IsRelevanceSort(searchOptions)) + { + newSearchOptions = searchOptions.CloneSqlSearchOptions(); + newSearchOptions.Sort = new (SearchParameterInfo searchParameterInfo, SortOrder sortOrder)[] + { + (SearchParameterInfo.ScoreSearchParameter, SortOrder.Ascending), + (SearchParameterInfo.ResourceTypeSearchParameter, SortOrder.Ascending), + (_fakeLastUpdate, SortOrder.Ascending), + }; + + return newSearchOptions; + } + if (searchOptions.ResourceVersionTypes.HasFlag(ResourceVersionType.History) && searchOptions.Sort.Any()) { // history is always sorted by _lastUpdated (except for export). @@ -1814,9 +1932,21 @@ private SqlSearchOptions UpdateSort(SqlSearchOptions searchOptions, Expression s return newSearchOptions; } + private static bool IsRelevanceSort(SqlSearchOptions searchOptions) + { + return searchOptions.PreparedVectorQuery != null && + (searchOptions.Sort.Count == 0 || IsScoreSort(searchOptions)); + } + + private static bool IsScoreSort(SearchOptions searchOptions) + { + return searchOptions.Sort.Count > 0 && searchOptions.Sort[0].searchParameterInfo.Name == SearchParameterNames.Score; + } + private void ReadWrapper( SqlDataReader reader, bool readIsHistory, + bool readSemanticEvidence, out short resourceTypeId, out string resourceId, out int version, @@ -1829,7 +1959,15 @@ private void ReadWrapper( out string searchParameterHash, out byte[] rawResourceBytes, out bool isInvisible, - out bool isHistory) + out bool isHistory, + out double? semanticDistance, + out int? semanticChunkOrdinal, + out string semanticChunkText, + out short? semanticSourceResourceTypeId, + out string semanticSourceResourceId, + out string semanticSourceResourceVersion, + out string semanticSourcePath, + out string semanticEvidenceJson) { resourceTypeId = reader.Read(VLatest.Resource.ResourceTypeId, 0); resourceId = reader.Read(VLatest.Resource.ResourceId, 1); @@ -1844,6 +1982,103 @@ private void ReadWrapper( rawResourceBytes = reader.GetSqlBytes(10).Value; isInvisible = rawResourceBytes.Length == 1 && rawResourceBytes[0] == 0xF; isHistory = readIsHistory && reader.FieldCount > 11 ? reader.Read(VLatest.Resource.IsHistory, 11) : false; + semanticDistance = readSemanticEvidence ? Convert.ToDouble(reader.GetValue(SemanticDistanceColumnName), CultureInfo.InvariantCulture) : null; + semanticChunkOrdinal = readSemanticEvidence ? Convert.ToInt32(reader.GetValue(SemanticChunkOrdinalColumnName), CultureInfo.InvariantCulture) : null; + semanticChunkText = readSemanticEvidence ? Convert.ToString(reader.GetValue(SemanticChunkTextColumnName), CultureInfo.InvariantCulture) : null; + semanticSourceResourceTypeId = readSemanticEvidence && !reader.IsDBNull(reader.GetOrdinal(SemanticSourceResourceTypeIdColumnName)) + ? Convert.ToInt16(reader.GetValue(SemanticSourceResourceTypeIdColumnName), CultureInfo.InvariantCulture) + : null; + semanticSourceResourceId = readSemanticEvidence && !reader.IsDBNull(reader.GetOrdinal(SemanticSourceResourceIdColumnName)) + ? Convert.ToString(reader.GetValue(SemanticSourceResourceIdColumnName), CultureInfo.InvariantCulture) + : null; + semanticSourceResourceVersion = readSemanticEvidence && !reader.IsDBNull(reader.GetOrdinal(SemanticSourceResourceVersionColumnName)) + ? Convert.ToString(reader.GetValue(SemanticSourceResourceVersionColumnName), CultureInfo.InvariantCulture) + : null; + semanticSourcePath = readSemanticEvidence && !reader.IsDBNull(reader.GetOrdinal(SemanticSourcePathColumnName)) + ? Convert.ToString(reader.GetValue(SemanticSourcePathColumnName), CultureInfo.InvariantCulture) + : null; + semanticEvidenceJson = readSemanticEvidence && !reader.IsDBNull(reader.GetOrdinal(SemanticEvidenceJsonColumnName)) + ? Convert.ToString(reader.GetValue(SemanticEvidenceJsonColumnName), CultureInfo.InvariantCulture) + : null; + } + + internal IReadOnlyList DeserializeSemanticEvidence( + string semanticEvidenceJson, + PreparedVectorSearchQuery preparedQuery, + ResourceWrapper resourceWrapper) + { + if (string.IsNullOrWhiteSpace(semanticEvidenceJson)) + { + return Array.Empty(); + } + + try + { + using JsonDocument document = JsonDocument.Parse(semanticEvidenceJson); + var evidenceItems = new List(); + foreach (JsonElement element in document.RootElement.EnumerateArray()) + { + string sourceResourceType = element.TryGetProperty("sourceResourceTypeId", out JsonElement sourceResourceTypeId) + ? _model.GetResourceTypeName(sourceResourceTypeId.GetInt16()) + : resourceWrapper.ResourceTypeName; + string sourceResourceId = element.TryGetProperty("sourceResourceId", out JsonElement sourceResourceIdElement) + ? sourceResourceIdElement.GetString() + : resourceWrapper.ResourceId; + string sourceResourceVersion = element.TryGetProperty("sourceResourceVersion", out JsonElement sourceResourceVersionElement) + ? sourceResourceVersionElement.GetString() + : resourceWrapper.Version; + string sourcePath = element.TryGetProperty("sourcePath", out JsonElement sourcePathElement) + ? sourcePathElement.GetString() + : preparedQuery.SearchParameter.Expression; + string witnessReference = null; + if (element.TryGetProperty("witnessResourceTypeId", out JsonElement witnessResourceTypeId) && + element.TryGetProperty("witnessResourceId", out JsonElement witnessResourceId) && + element.TryGetProperty("witnessResourceVersion", out JsonElement witnessResourceVersion)) + { + witnessReference = new ResourceKey( + _model.GetResourceTypeName(witnessResourceTypeId.GetInt16()), + witnessResourceId.GetString(), + witnessResourceVersion.GetInt32().ToString(CultureInfo.InvariantCulture)).ToString(); + } + + evidenceItems.Add(new SemanticSearchEvidence( + element.GetProperty("text").GetString(), + element.GetProperty("chunkOrdinal").GetInt32(), + NormalizeCosineDistance(element.GetProperty("distance").GetDouble()), + preparedQuery.SearchParameter.Url, + new ResourceKey(sourceResourceType, sourceResourceId, sourceResourceVersion).ToString(), + sourcePath, + witnessReference: witnessReference)); + } + + return evidenceItems; + } + catch (Exception exception) when (exception is JsonException or InvalidOperationException or KeyNotFoundException or FormatException) + { + _logger.LogWarning("Unable to deserialize semantic evidence; evidence will be discarded."); + return Array.Empty(); + } + } + + private static void AssignEvidenceRanks(List matchedResources) + { + IReadOnlyList> rankedEvidence = SemanticSearchEvidenceRanker.AssignRanks( + matchedResources.Select(result => result.EvidenceItems).ToList()); + + for (int index = 0; index < matchedResources.Count; index++) + { + SearchResultEntry result = matchedResources[index]; + matchedResources[index] = new SearchResultEntry( + result.Resource, + result.SearchEntryMode, + result.Score, + evidenceItems: rankedEvidence[index]); + } + } + + private static decimal NormalizeCosineDistance(double distance) + { + return (decimal)Math.Clamp(1.0 - (distance / 2.0), 0.0, 1.0); } [Conditional("DEBUG")] @@ -2098,6 +2333,7 @@ await _sqlRetryService.ExecuteSql( ReadWrapper( reader, exportTimeTravel, + false, out short resourceTypeId, out string resourceId, out int version, @@ -2110,7 +2346,15 @@ await _sqlRetryService.ExecuteSql( out string searchParameterHash, out byte[] rawResourceBytes, out bool isInvisible, - out bool isHistory); + out bool isHistory, + out double? _, + out int? _, + out string _, + out short? _, + out string _, + out string _, + out string _, + out string _); if (isInvisible) { @@ -3009,6 +3253,18 @@ private async Task Init(ISqlRetryService sqlRetryService, ILogger + { + public static readonly VectorSearchPresenceVisitor Instance = new VectorSearchPresenceVisitor(); + + private VectorSearchPresenceVisitor() + : base((left, right) => left || right) + { + } + + public override bool VisitVectorSearch(VectorSearchExpression expression, object context) => true; + } + private class Token { internal Token(string code, int? systemId, string systemValue) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSortingValidator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSortingValidator.cs index 42a3729a96..aab280e9b4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSortingValidator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSortingValidator.cs @@ -66,6 +66,7 @@ public bool ValidateSorting(IReadOnlyList<(SearchParameterInfo searchParameter, switch (sorting) { case { Count: 0 }: + case { Count: 1 } when sorting[0].searchParameter.Name == SearchParameterNames.Score: case { Count: 1 } when _schemaInformation.Current >= SchemaVersionConstants.AddMinMaxForDateAndStringSearchParamVersion && SupportedSortParamTypes.Contains(sorting[0].searchParameter.Type): errorMessages = Array.Empty(); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index 39d918e025..00b5ce2d25 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -32,7 +32,9 @@ using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Features.Search.Registry; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.SqlServer.Features.Schema; using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration; using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration.Merge; @@ -69,6 +71,7 @@ internal class SqlServerFhirDataStore : IFhirDataStore, IProvideCapability private readonly SchemaInformation _schemaInformation; private readonly IModelInfoProvider _modelInfoProvider; private readonly IImportErrorSerializer _importErrorSerializer; + private readonly IVectorSearchIndexer _vectorSearchIndexer; private static CachedParameter _ignoreInputLastUpdated; private static CachedParameter _ignoreInputVersion; private static CachedParameter _rawResourceDeduping; @@ -88,7 +91,8 @@ public SqlServerFhirDataStore( IModelInfoProvider modelInfoProvider, RequestContextAccessor requestContextAccessor, IImportErrorSerializer importErrorSerializer, - SqlStoreClient storeClient) + SqlStoreClient storeClient, + IVectorSearchIndexer vectorSearchIndexer = null) { _model = EnsureArg.IsNotNull(model, nameof(model)); _searchParameterTypeMap = EnsureArg.IsNotNull(searchParameterTypeMap, nameof(searchParameterTypeMap)); @@ -104,6 +108,7 @@ public SqlServerFhirDataStore( _modelInfoProvider = EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider)); _requestContextAccessor = EnsureArg.IsNotNull(requestContextAccessor, nameof(requestContextAccessor)); _importErrorSerializer = EnsureArg.IsNotNull(importErrorSerializer, nameof(importErrorSerializer)); + _vectorSearchIndexer = vectorSearchIndexer; _memoryStreamManager = new RecyclableMemoryStreamManager(); @@ -456,6 +461,13 @@ private async Task MergeInternalAsync(IReadOnlyList _.PendingSearchParameterStatus != null).Select(_ => _.PendingSearchParameterStatus).ToList(); if (mergeWrappersWithVersions.Count > 0 || pendingStatuses.Count > 0) // Do not call DB with empty input { + if (_vectorSearchIndexer != null && mergeWrappersWithVersions.Count > 0) + { + await _vectorSearchIndexer.IndexAsync( + mergeWrappersWithVersions.Select(item => item.Wrapper.ResourceWrapper).ToList(), + cancellationToken); + } + await using (new Timer(async _ => await _sqlStoreClient.MergeResourcesPutTransactionHeartbeatAsync(transactionId, MergeResourcesTransactionHeartbeatPeriod, cancellationToken), null, TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(100) / 100.0 * MergeResourcesTransactionHeartbeatPeriod.TotalSeconds), MergeResourcesTransactionHeartbeatPeriod)) { var retries = 0; @@ -818,15 +830,32 @@ internal async Task MergeResourcesWrapperAsync(long transactionId, bool singleTr using var cmd = new SqlCommand(); //// Do not use auto generated tvp generator as it does not allow to skip compartment tvp and paramters with default values cmd.CommandType = CommandType.StoredProcedure; + bool enqueueVectorSearchSourceRefresh = ShouldEnqueueVectorSearchSourceRefresh(_vectorSearchIndexer, _schemaInformation.Current); if (pendingStatuses?.Count > 0) { - cmd.CommandText = "dbo.MergeResourcesAndSearchParams"; + if (enqueueVectorSearchSourceRefresh) + { + cmd.CommandText = "dbo.MergeResourcesAndSearchParamsWithVectorSearchSourceRefresh"; + } + else + { + cmd.CommandText = "dbo.MergeResourcesAndSearchParams"; + } + new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(pendingStatuses)); } else { - cmd.CommandText = "dbo.MergeResources"; + if (enqueueVectorSearchSourceRefresh) + { + cmd.CommandText = "dbo.MergeResourcesWithVectorSearchSourceRefresh"; + } + else + { + cmd.CommandText = "dbo.MergeResources"; + } + cmd.Parameters.AddWithValue("@SingleTransaction", singleTransaction); } @@ -843,6 +872,7 @@ internal async Task MergeResourcesWrapperAsync(long transactionId, bool singleTr new NumberSearchParamListTableValuedParameterDefinition("@NumberSearchParams").AddParameter(cmd.Parameters, new NumberSearchParamListRowGenerator(_model, _searchParameterTypeMap).GenerateRows(mergeWrappers)); new QuantitySearchParamListTableValuedParameterDefinition("@QuantitySearchParams").AddParameter(cmd.Parameters, new QuantitySearchParamListRowGenerator(_model, _searchParameterTypeMap).GenerateRows(mergeWrappers)); new DateTimeSearchParamListTableValuedParameterDefinition("@DateTimeSearchParms").AddParameter(cmd.Parameters, new DateTimeSearchParamListRowGenerator(_model, _searchParameterTypeMap).GenerateRows(mergeWrappers)); + new VectorSearchParamListTableValuedParameterDefinition("@VectorSearchParams").AddParameter(cmd.Parameters, new VectorSearchParamListRowGenerator(_model).GenerateRows(mergeWrappers)); new ReferenceTokenCompositeSearchParamListTableValuedParameterDefinition("@ReferenceTokenCompositeSearchParams").AddParameter(cmd.Parameters, new ReferenceTokenCompositeSearchParamListRowGenerator(_model, new ReferenceSearchParamListRowGenerator(_model, _searchParameterTypeMap), new TokenSearchParamListRowGenerator(_model, _searchParameterTypeMap), _searchParameterTypeMap).GenerateRows(mergeWrappers)); new TokenTokenCompositeSearchParamListTableValuedParameterDefinition("@TokenTokenCompositeSearchParams").AddParameter(cmd.Parameters, new TokenTokenCompositeSearchParamListRowGenerator(_model, new TokenSearchParamListRowGenerator(_model, _searchParameterTypeMap), _searchParameterTypeMap).GenerateRows(mergeWrappers)); new TokenDateTimeCompositeSearchParamListTableValuedParameterDefinition("@TokenDateTimeCompositeSearchParams").AddParameter(cmd.Parameters, new TokenDateTimeCompositeSearchParamListRowGenerator(_model, new TokenSearchParamListRowGenerator(_model, _searchParameterTypeMap), new DateTimeSearchParamListRowGenerator(_model, _searchParameterTypeMap), _searchParameterTypeMap).GenerateRows(mergeWrappers)); @@ -936,7 +966,14 @@ public async Task GetAsync(ResourceKey key, CancellationToken c public async Task HardDeleteAsync(ResourceKey key, bool keepCurrentVersion, bool allowPartialSuccess, CancellationToken cancellationToken) { - await _sqlStoreClient.HardDeleteAsync(_model.GetResourceTypeId(key.ResourceType), key.Id, keepCurrentVersion, _coreFeatures.SupportsResourceChangeCapture, cancellationToken); + bool enqueueVectorSearchSourceRefresh = ShouldEnqueueVectorSearchSourceRefresh(_vectorSearchIndexer, _schemaInformation.Current); + await _sqlStoreClient.HardDeleteAsync( + _model.GetResourceTypeId(key.ResourceType), + key.Id, + keepCurrentVersion, + _coreFeatures.SupportsResourceChangeCapture, + enqueueVectorSearchSourceRefresh, + cancellationToken); } public async Task BulkUpdateSearchParameterIndicesAsync(IReadOnlyCollection resources, CancellationToken cancellationToken) @@ -951,8 +988,10 @@ public async Task BulkUpdateSearchParameterIndicesAsync(IReadOnlyCollection new MergeResourceWrapper(_, false, false)).ToList(); + var vectorMergeWrappers = mergeWrappers.Where(resource => resource.ResourceWrapper.VectorSearchIndicesUpdated).ToList(); + bool updateVectorSearchIndices = ShouldUpdateVectorSearchIndices(resources, _schemaInformation.Current); - using var cmd = new SqlCommand("dbo.UpdateResourceSearchParams") { CommandType = CommandType.StoredProcedure, CommandTimeout = 300 + (int)(3600.0 / 10000 * mergeWrappers.Count) }; + using SqlCommand cmd = CreateBulkUpdateSearchParameterIndicesCommand(updateVectorSearchIndices, mergeWrappers.Count); new ResourceListTableValuedParameterDefinition("@Resources").AddParameter(cmd.Parameters, new ResourceListRowGenerator(_model, _compressedRawResourceConverter).GenerateRows(mergeWrappers)); new ResourceWriteClaimListTableValuedParameterDefinition("@ResourceWriteClaims").AddParameter(cmd.Parameters, new ResourceWriteClaimListRowGenerator(_model, _searchParameterTypeMap).GenerateRows(mergeWrappers)); new ReferenceSearchParamListTableValuedParameterDefinition("@ReferenceSearchParams").AddParameter(cmd.Parameters, new ReferenceSearchParamListRowGenerator(_model, _searchParameterTypeMap).GenerateRows(mergeWrappers)); @@ -969,6 +1008,12 @@ public async Task BulkUpdateSearchParameterIndicesAsync(IReadOnlyCollection resources, int? currentSchemaVersion) + { + return currentSchemaVersion >= SchemaVersionConstants.VectorSearchReindexVersion && resources.Any(resource => resource.VectorSearchIndicesUpdated); + } + + internal static bool ShouldEnqueueVectorSearchSourceRefresh(IVectorSearchIndexer vectorSearchIndexer, int? currentSchemaVersion) + { + return vectorSearchIndexer != null && currentSchemaVersion >= SchemaVersionConstants.VectorSearchSourceRefreshVersion; + } + + internal static SqlCommand CreateBulkUpdateSearchParameterIndicesCommand(bool updateVectorSearchIndices, int resourceCount) + { + var command = updateVectorSearchIndices + ? new SqlCommand("dbo.UpdateResourceSearchParamsWithVectors") + : new SqlCommand("dbo.UpdateResourceSearchParams"); + command.CommandType = CommandType.StoredProcedure; + command.CommandTimeout = 300 + (int)(3600.0 / 10000 * resourceCount); + return command; + } + private static string RemoveTrailingZerosFromMillisecondsForAGivenDate(DateTimeOffset date) { // 0000000+ -> +, 0010000+ -> 001+, 0100000+ -> 01+, 0180000+ -> 018+, 1000000 -> 1+, 1100000+ -> 11+, 1010000+ -> 101+ diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs index a1caf6cce8..24fece0a5b 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs @@ -43,9 +43,18 @@ public SqlStoreClient(ISqlRetryService sqlRetryService, ILogger _schemaInformation = schemaInformation; } - public async Task HardDeleteAsync(short resourceTypeId, string resourceId, bool keepCurrentVersion, bool isResourceChangeCaptureEnabled, CancellationToken cancellationToken) + public async Task HardDeleteAsync(short resourceTypeId, string resourceId, bool keepCurrentVersion, bool isResourceChangeCaptureEnabled, bool enqueueVectorSearchSourceRefresh, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.HardDeleteResource", CommandType = CommandType.StoredProcedure }; + using var cmd = new SqlCommand() { CommandType = CommandType.StoredProcedure }; + if (enqueueVectorSearchSourceRefresh) + { + cmd.CommandText = "dbo.HardDeleteResourceWithVectorSearchSourceRefresh"; + } + else + { + cmd.CommandText = "dbo.HardDeleteResource"; + } + cmd.Parameters.AddWithValue("@ResourceTypeId", resourceTypeId); cmd.Parameters.AddWithValue("@ResourceId", resourceId); cmd.Parameters.AddWithValue("@KeepCurrentVersion", keepCurrentVersion); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlVectorSearchSourceDependencyStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlVectorSearchSourceDependencyStore.cs new file mode 100644 index 0000000000..2da425c11c --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlVectorSearchSourceDependencyStore.cs @@ -0,0 +1,59 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Data; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.SqlServer.Features.Client; +using Microsoft.Health.SqlServer.Features.Storage; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Storage +{ + internal sealed class SqlVectorSearchSourceDependencyStore : IVectorSearchSourceDependencyStore + { + private readonly SqlServerFhirModel _model; + private readonly ISqlRetryService _sqlRetryService; + private readonly ILogger _logger; + + public SqlVectorSearchSourceDependencyStore( + SqlServerFhirModel model, + ISqlRetryService sqlRetryService, + ILogger logger) + { + _model = EnsureArg.IsNotNull(model, nameof(model)); + _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + } + + public async Task> GetDependentResourceKeysAsync( + string sourceResourceType, + string sourceResourceId, + CancellationToken cancellationToken) + { + EnsureArg.IsNotNullOrWhiteSpace(sourceResourceType, nameof(sourceResourceType)); + EnsureArg.IsNotNullOrWhiteSpace(sourceResourceId, nameof(sourceResourceId)); + + using var command = new SqlCommand("dbo.GetVectorSearchSourceDependencies") + { + CommandType = CommandType.StoredProcedure, + }; + + command.Parameters.AddWithValue("@SourceResourceTypeId", _model.GetResourceTypeId(sourceResourceType)); + command.Parameters.AddWithValue("@SourceResourceId", sourceResourceId); + + return await command.ExecuteReaderAsync( + _sqlRetryService, + reader => new ResourceKey(_model.GetResourceTypeName(reader.GetInt16(0)), reader.GetString(1)), + _logger, + cancellationToken); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/MergeSearchParameterRowGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/MergeSearchParameterRowGenerator.cs index 63fd51be91..b1582421e9 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/MergeSearchParameterRowGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/MergeSearchParameterRowGenerator.cs @@ -42,7 +42,7 @@ public virtual IEnumerable GenerateRows(IReadOnlyList _searchParameterTypeMap.GetSearchValueType(e)), + merge.ResourceWrapper.SearchIndices?.Where(e => e.SearchParameter.VectorConfig == null).ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)), merge.ResourceWrapper.LastModifiedClaims); var resultsForDedupping = new HashSet(); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/ResourceWriteClaimListRowGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/ResourceWriteClaimListRowGenerator.cs index 659b923e50..ceed6363af 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/ResourceWriteClaimListRowGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/ResourceWriteClaimListRowGenerator.cs @@ -34,7 +34,7 @@ public IEnumerable GenerateRows(IReadOnlyList _searchParameterTypeMap.GetSearchValueType(e)), + resource.SearchIndices?.Where(e => e.SearchParameter.VectorConfig == null).ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)), resource.LastModifiedClaims); IReadOnlyCollection> writeClaims = resourceMetadata.WriteClaims; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/VectorSearchParamListRowGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/VectorSearchParamListRowGenerator.cs new file mode 100644 index 0000000000..a88eb905de --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/Merge/VectorSearchParamListRowGenerator.cs @@ -0,0 +1,69 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Linq; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; +using Microsoft.Health.Fhir.SqlServer.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration.Merge; +using Microsoft.Health.SqlServer.Features.Schema.Model; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration +{ + internal sealed class VectorSearchParamListRowGenerator : ITableValuedParameterRowGenerator, VectorSearchParamListRow> + { + private readonly SqlServerFhirModel _model; + + public VectorSearchParamListRowGenerator(SqlServerFhirModel model) + { + _model = EnsureArg.IsNotNull(model, nameof(model)); + } + + public IEnumerable GenerateRows(IReadOnlyList resources) + { + EnsureArg.IsNotNull(resources, nameof(resources)); + + foreach (MergeResourceWrapper merge in resources.Where(resource => !resource.ResourceWrapper.IsHistory)) + { + short resourceTypeId = _model.GetResourceTypeId(merge.ResourceWrapper.ResourceTypeName); + + foreach (VectorSearchIndexEntry vectorIndex in merge.ResourceWrapper.VectorSearchIndices) + { + short searchParamId = _model.GetSearchParamId(vectorIndex.SearchParameter.Url); + + foreach (VectorSearchChunk chunk in vectorIndex.Chunks) + { + yield return new VectorSearchParamListRow( + resourceTypeId, + merge.ResourceWrapper.ResourceSurrogateId, + searchParamId, + checked((short)chunk.ChunkOrdinal), + vectorIndex.EmbeddingModelId, + chunk.ChunkText, + ToArray(chunk.SourceTextHash), + _model.GetResourceTypeId(chunk.SourceResourceType ?? merge.ResourceWrapper.ResourceTypeName), + chunk.SourceResourceId ?? merge.ResourceWrapper.ResourceId, + chunk.SourceResourceVersion ?? merge.ResourceWrapper.Version, + chunk.SourcePath ?? vectorIndex.SearchParameter.Expression ?? vectorIndex.SearchParameter.Code, + SqlVectorFormatter.Format(chunk.Embedding)); + } + } + } + } + + private static byte[] ToArray(IReadOnlyList source) + { + var result = new byte[source.Count]; + for (int index = 0; index < result.Length; index++) + { + result[index] = source[index]; + } + + return result; + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs index 0b0e17b1e1..807406d65c 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs @@ -11,6 +11,7 @@ using EnsureThat; using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration.Merge; @@ -20,16 +21,23 @@ internal sealed class TransactionWatchdog : Watchdog { private readonly SqlServerFhirDataStore _store; private readonly IResourceWrapperFactory _factory; + private readonly IVectorSearchIndexer _vectorSearchIndexer; private readonly ILogger _logger; private const string AdvancedVisibilityTemplate = "TransactionWatchdog advanced visibility on {Transactions} transactions."; private const string FoundTemplate = "TransactionWatchdog found {Transactions} timed out transactions."; - public TransactionWatchdog(SqlServerFhirDataStore store, IResourceWrapperFactory factory, ISqlRetryService sqlRetryService, ILogger logger) + public TransactionWatchdog( + SqlServerFhirDataStore store, + IResourceWrapperFactory factory, + ISqlRetryService sqlRetryService, + ILogger logger, + IVectorSearchIndexer vectorSearchIndexer = null) : base(sqlRetryService, logger) { _store = EnsureArg.IsNotNull(store, nameof(store)); _factory = EnsureArg.IsNotNull(factory, nameof(factory)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _vectorSearchIndexer = vectorSearchIndexer; } internal TransactionWatchdog() @@ -83,6 +91,11 @@ protected override async Task RunWorkAsync(CancellationToken cancellationToken) _factory.Update(resource); } + if (_vectorSearchIndexer != null) + { + await _vectorSearchIndexer.IndexAsync(resources, cancellationToken); + } + await _store.MergeResourcesWrapperAsync(tranId, false, resources.Select(x => new MergeResourceWrapper(x, true, true)).ToList(), false, 0, null, cancellationToken); await _store.StoreClient.MergeResourcesCommitTransactionAsync(tranId, null, cancellationToken); _logger.LogWarning("TransactionWatchdog committed transaction={Transaction}, resources={Resources}", tranId, resources.Count); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index 9c6dfd9a2e..3730315cc0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -1,7 +1,7 @@  - 116 + 119 Features\Schema\Migrations\$(LatestSchemaVersion).sql LatestSchemaVersion-$(LatestSchemaVersion) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs index 12fe9334b3..1cba52fe51 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs @@ -23,6 +23,7 @@ using Microsoft.Health.Fhir.Core.Features.Parameters; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Features.Search.Registry; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Logging.Metrics; using Microsoft.Health.Fhir.Core.Messages.Search; using Microsoft.Health.Fhir.Core.Messages.Storage; @@ -34,6 +35,7 @@ using Microsoft.Health.Fhir.SqlServer.Features.Search; using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions; using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors; +using Microsoft.Health.Fhir.SqlServer.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.SqlServer.Features.Storage.Registry; using Microsoft.Health.Fhir.SqlServer.Features.Watchdogs; @@ -97,6 +99,11 @@ public static IFhirServerBuilder AddSqlServer(this IFhirServerBuilder fhirServer .AsSelf() .AsImplementedInterfaces(); + services.Add() + .Scoped() + .AsSelf() + .AsService(); + services.Add() .Scoped() .AsSelf() @@ -196,6 +203,10 @@ public static IFhirServerBuilder AddSqlServer(this IFhirServerBuilder fhirServer .Singleton() .AsSelf(); + services.Add() + .Scoped() + .AsImplementedInterfaces(); + services.AddSingleton>(); services.Add() diff --git a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/SemanticSearchTestFixture.cs b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/SemanticSearchTestFixture.cs new file mode 100644 index 0000000000..47cdb593b3 --- /dev/null +++ b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/SemanticSearchTestFixture.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; + +namespace Microsoft.Health.Fhir.Tests.E2E.Rest +{ + public sealed class SemanticSearchTestFixture : HttpIntegrationTestFixture + { + public SemanticSearchTestFixture(DataStore dataStore, Format format, TestFhirServerFactory testFhirServerFactory) + : base(dataStore, format, testFhirServerFactory) + { + } + } +} diff --git a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/SemanticSearchTestParameterResolver.cs b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/SemanticSearchTestParameterResolver.cs new file mode 100644 index 0000000000..02637c3999 --- /dev/null +++ b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/SemanticSearchTestParameterResolver.cs @@ -0,0 +1,76 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Hl7.Fhir.Model; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using CoreSearchParamType = Microsoft.Health.Fhir.ValueSets.SearchParamType; + +namespace Microsoft.Health.Fhir.Tests.E2E.Rest +{ + internal sealed class SemanticSearchTestParameterResolver : IVectorSearchParameterResolver + { + internal static readonly Uri DocumentReferenceCanonical = new Uri("https://example.org/fhir/SearchParameter/e2e-document-reference-semantic-text"); + internal static readonly Uri ObservationCanonical = new Uri("https://example.org/fhir/SearchParameter/e2e-observation-semantic-text"); + internal static readonly Uri DiagnosticReportCanonical = new Uri("https://example.org/fhir/SearchParameter/e2e-diagnostic-report-semantic-text"); + internal static readonly Uri CoverageCanonical = new Uri("https://example.org/fhir/SearchParameter/e2e-coverage-semantic-text"); + + private readonly IReadOnlyDictionary _searchParameters = new Dictionary(StringComparer.Ordinal) + { + [ResourceType.DocumentReference.ToString()] = new SearchParameterInfo( + name: "DocumentReferenceSemantic", + code: "semantic-text", + searchParamType: CoreSearchParamType.Special, + url: DocumentReferenceCanonical, + expression: "DocumentReference.content.attachment.url.toString()", + baseResourceTypes: new[] { ResourceType.DocumentReference.ToString() }, + vectorConfig: new VectorSearchParameterConfig { SourceStrategy = VectorTextSourceStrategy.LocalBinaryReference }), + [ResourceType.Observation.ToString()] = new SearchParameterInfo( + name: "ObservationSemantic", + code: "semantic-text", + searchParamType: CoreSearchParamType.Special, + url: ObservationCanonical, + expression: "Observation.note.text", + baseResourceTypes: new[] { ResourceType.Observation.ToString() }, + vectorConfig: new VectorSearchParameterConfig()), + [ResourceType.DiagnosticReport.ToString()] = new SearchParameterInfo( + name: "DiagnosticReportSemantic", + code: "semantic-text", + searchParamType: CoreSearchParamType.Special, + url: DiagnosticReportCanonical, + expression: "DiagnosticReport.conclusion", + baseResourceTypes: new[] { ResourceType.DiagnosticReport.ToString() }, + vectorConfig: new VectorSearchParameterConfig()), + [ResourceType.Coverage.ToString()] = new SearchParameterInfo( + name: "CoverageSemantic", + code: "semantic-text", + searchParamType: CoreSearchParamType.Special, + url: CoverageCanonical, + expression: "Coverage.class.name", + baseResourceTypes: new[] { ResourceType.Coverage.ToString() }, + vectorConfig: new VectorSearchParameterConfig()), + }; + + public IReadOnlyList GetSearchParameters(string resourceType) + { + return _searchParameters.TryGetValue(resourceType, out SearchParameterInfo searchParameter) + ? new[] { searchParameter } + : Array.Empty(); + } + + public IReadOnlyList GetIndexingSearchParameters(string resourceType) + { + return GetSearchParameters(resourceType); + } + + public SearchParameterInfo GetSearchParameter(Uri canonicalUri) + { + return _searchParameters.Values.Single(searchParameter => searchParameter.Url == canonicalUri); + } + } +} diff --git a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/SemanticSearchTests.cs b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/SemanticSearchTests.cs new file mode 100644 index 0000000000..7521a478b5 --- /dev/null +++ b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/SemanticSearchTests.cs @@ -0,0 +1,335 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Hl7.Fhir.Model; +using Hl7.Fhir.Serialization; +using Microsoft.Health.Extensions.Xunit; +using Microsoft.Health.Fhir.Client; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using Microsoft.Health.Fhir.Tests.E2E.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Health.Fhir.Tests.E2E.Rest +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + public sealed class SemanticSearchTests : IClassFixture + { + private const string Query = "difficulty breathing after exercise"; + private readonly TestFhirClient _client; + + public SemanticSearchTests(SemanticSearchTestFixture fixture) + { + _client = fixture.TestFhirClient; + } + + [Fact] + public async Task GivenPatientResources_WhenSemanticSearchIsInvoked_ThenMixedRankedBundleContainsEvidence() + { + await EnsureCoverageSearchParameterIsEnabledAsync(); + + Patient patient = await CreateAsync(new Patient { Active = true }); + Patient otherPatient = await CreateAsync(new Patient { Active = true }); + Organization payor = await CreateAsync(new Organization { Active = true, Name = "Semantic search test payor" }); + Binary binary = await CreateAsync(new Binary + { + ContentType = "text/plain", + Data = Encoding.UTF8.GetBytes("The patient reports shortness of breath while climbing stairs."), + }); + DocumentReference documentReference = await CreateAsync(new DocumentReference + { + Status = DocumentReferenceStatus.Current, + Subject = new ResourceReference($"Patient/{patient.Id}"), + Content = + { + new DocumentReference.ContentComponent + { + Attachment = new Attachment + { + ContentType = "text/plain", + Url = $"Binary/{binary.Id}", + }, + }, + }, + }); + Observation observation = await CreateAsync(new Observation + { + Status = ObservationStatus.Final, + Code = new CodeableConcept("http://loinc.org", "75325-1", "Symptom"), + Subject = new ResourceReference($"Patient/{patient.Id}"), + Note = { new Annotation { Text = Query } }, + }); + DiagnosticReport diagnosticReport = await CreateAsync(new DiagnosticReport + { + Status = DiagnosticReport.DiagnosticReportStatus.Final, + Code = new CodeableConcept("http://loinc.org", "19868-9", "Pulmonary function study"), + Subject = new ResourceReference($"Patient/{patient.Id}"), + Conclusion = "Pulmonary testing indicates exertional airflow limitation.", + }); + Coverage coverage = await CreateAsync(CreateCoverage(patient.Id, payor.Id, Query)); + Observation otherPatientObservation = await CreateAsync(new Observation + { + Status = ObservationStatus.Final, + Code = new CodeableConcept("http://loinc.org", "75325-1", "Symptom"), + Subject = new ResourceReference($"Patient/{otherPatient.Id}"), + Note = { new Annotation { Text = Query } }, + }); + Coverage otherPatientCoverage = await CreateAsync(CreateCoverage(otherPatient.Id, payor.Id, Query)); + var parameters = new Parameters + { + Parameter = + { + new Parameters.ParameterComponent { Name = "query", Value = new FhirString(Query) }, + new Parameters.ParameterComponent { Name = "count", Value = new Integer(10) }, + }, + }; + + using FhirResponse response = await _client.PostAsync( + $"Patient/{patient.Id}/$semantic-search", + parameters.ToJson()); + + Bundle bundle = Assert.IsType(response.Resource); + Assert.Equal(4, bundle.Total); + Assert.Equal( + new[] { documentReference.Id, observation.Id, diagnosticReport.Id, coverage.Id }.OrderBy(id => id), + bundle.Entry.Select(entry => entry.Resource.Id).OrderBy(id => id)); + Assert.DoesNotContain(bundle.Entry, entry => entry.Resource.Id == otherPatientObservation.Id); + Assert.DoesNotContain(bundle.Entry, entry => entry.Resource.Id == otherPatientCoverage.Id); + Assert.Equal( + bundle.Entry.Select(entry => entry.Search.Score).OrderByDescending(score => score), + bundle.Entry.Select(entry => entry.Search.Score)); + + Bundle.EntryComponent observationEntry = Assert.Single(bundle.Entry, entry => entry.Resource.Id == observation.Id); + Assert.Equal(1m, observationEntry.Search.Score); + Bundle.EntryComponent coverageEntry = Assert.Single(bundle.Entry, entry => entry.Resource.Id == coverage.Id); + Assert.Equal(1m, coverageEntry.Search.Score); + + Bundle.EntryComponent documentReferenceEntry = Assert.Single(bundle.Entry, entry => entry.Resource.Id == documentReference.Id); + Extension documentEvidence = Assert.Single( + documentReferenceEntry.Search.Extension, + extension => extension.Url == SemanticSearchEvidence.ExtensionUrl); + Assert.Equal( + $"Binary/{binary.Id}/_history/{binary.Meta.VersionId}", + ((ResourceReference)documentEvidence.Extension.Single(extension => extension.Url == SemanticSearchEvidence.SourceExtensionUrl).Value).Reference); + Assert.Equal( + "Binary.data", + ((FhirString)documentEvidence.Extension.Single(extension => extension.Url == SemanticSearchEvidence.SourcePathExtensionUrl).Value).Value); + Assert.Equal( + "The patient reports shortness of breath while climbing stairs.", + ((FhirString)documentEvidence.Extension.Single(extension => extension.Url == SemanticSearchEvidence.TextExtensionUrl).Value).Value); + + var coverageOnlyParameters = new Parameters + { + Parameter = + { + new Parameters.ParameterComponent { Name = "query", Value = new FhirString(Query) }, + new Parameters.ParameterComponent { Name = "count", Value = new Integer(10) }, + new Parameters.ParameterComponent { Name = "type", Value = new Code(ResourceType.Coverage.ToString()) }, + }, + }; + + using FhirResponse coverageOnlyResponse = await _client.PostAsync( + $"Patient/{patient.Id}/$semantic-search", + coverageOnlyParameters.ToJson()); + + Bundle coverageOnlyBundle = Assert.IsType(coverageOnlyResponse.Resource); + Bundle.EntryComponent onlyCoverageEntry = Assert.Single(coverageOnlyBundle.Entry); + Assert.Equal(coverage.Id, onlyCoverageEntry.Resource.Id); + } + + [Fact] + public async Task GivenDocumentReferenceBeforeBinaryInTransaction_WhenSemanticSearchIsInvoked_ThenDocumentReferenceContainsBinaryEvidence() + { + // Arrange + string suffix = Guid.NewGuid().ToString("N"); + string documentReferenceId = $"semantic-document-{suffix}"; + string binaryId = $"semantic-binary-{suffix}"; + string passage = $"{Query} transaction {suffix}"; + var transaction = new Bundle + { + Type = Bundle.BundleType.Transaction, + Entry = + { + new Bundle.EntryComponent + { + Resource = new DocumentReference + { + Id = documentReferenceId, + Status = DocumentReferenceStatus.Current, + Content = + { + new DocumentReference.ContentComponent + { + Attachment = new Attachment + { + ContentType = "text/plain", + Url = $"Binary/{binaryId}", + }, + }, + }, + }, + Request = new Bundle.RequestComponent + { + Method = Bundle.HTTPVerb.PUT, + Url = $"DocumentReference/{documentReferenceId}", + }, + }, + new Bundle.EntryComponent + { + Resource = new Binary + { + Id = binaryId, + ContentType = "text/plain", + Data = Encoding.UTF8.GetBytes(passage), + }, + Request = new Bundle.RequestComponent + { + Method = Bundle.HTTPVerb.PUT, + Url = $"Binary/{binaryId}", + }, + }, + }, + }; + + // Act + using FhirResponse transactionResponse = await _client.PostBundleAsync( + transaction, + new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Sequential }); + Bundle searchResult = await _client.SearchAsync( + ResourceType.DocumentReference, + $"semantic-text={Uri.EscapeDataString(passage)}&_count=1"); + + // Assert + Assert.All(transactionResponse.Resource.Entry, entry => Assert.Equal("201", entry.Response.Status)); + Bundle.EntryComponent resultEntry = Assert.Single( + searchResult.Entry, + entry => entry.Search.Mode == Bundle.SearchEntryMode.Match && entry.Resource.Id == documentReferenceId); + string binaryReference = $"Binary/{binaryId}/_history/1"; + Assert.Contains( + resultEntry.Search.Extension, + extension => + extension.Url == SemanticSearchEvidence.ExtensionUrl && + extension.Extension.Any(component => + component.Url == SemanticSearchEvidence.SourceExtensionUrl && + component.Value is ResourceReference source && + source.Reference == binaryReference) && + extension.Extension.Any(component => + component.Url == SemanticSearchEvidence.TextExtensionUrl && + component.Value is FhirString text && + text.Value == passage)); + } + + private async Task EnsureCoverageSearchParameterIsEnabledAsync() + { + var searchParameter = new SearchParameter + { + Id = "coverage-semantic", + Url = SemanticSearchTestParameterResolver.CoverageCanonical.ToString(), + Name = "CoverageSemantic", + Status = PublicationStatus.Active, + Code = "semantic-text", + Type = SearchParamType.Special, + Expression = "Coverage.class.name", + Description = new Markdown("Semantic text in the Coverage plan name."), + Base = new ResourceType?[] { ResourceType.Coverage }, + Extension = + { + new Extension + { + Url = VectorSearchParameterConfig.ExtensionUrl, + Extension = + { + new Extension(VectorSearchParameterConfig.SourceStrategyExtensionUrl, new Code("directText")), + new Extension(VectorSearchParameterConfig.ExtractionPolicyExtensionUrl, new Code("perValueRow")), + }, + }, + }, + }; + + var updateStopwatch = Stopwatch.StartNew(); + while (true) + { + try + { + using FhirResponse updateResponse = await _client.UpdateAsync(searchParameter); + break; + } + catch (FhirClientException exception) when ( + exception.StatusCode == HttpStatusCode.Conflict && + updateStopwatch.Elapsed < TimeSpan.FromMinutes(2)) + { + exception.Dispose(); + await Task.Delay(TimeSpan.FromSeconds(1)); + } + } + + (_, Uri jobUri) = await _client.PostReindexJobAsync(new Parameters()); + var stopwatch = Stopwatch.StartNew(); + + while (stopwatch.Elapsed < TimeSpan.FromMinutes(2)) + { + using FhirResponse jobResponse = await _client.CheckJobAsync(jobUri); + DataType statusValue = jobResponse.Resource?.Parameter?.FirstOrDefault(parameter => parameter.Name == "status")?.Value; + string status = statusValue switch + { + Code code => code.Value, + FhirString text => text.Value, + _ => null, + }; + + if (string.Equals(status, "Completed", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + Assert.False( + string.Equals(status, "Failed", StringComparison.OrdinalIgnoreCase) || + string.Equals(status, "Canceled", StringComparison.OrdinalIgnoreCase), + $"Coverage SearchParameter reindex ended with status '{status}'."); + await Task.Delay(TimeSpan.FromSeconds(1)); + } + + Assert.Fail("Coverage SearchParameter reindex did not complete within two minutes."); + } + + private static Coverage CreateCoverage(string patientId, string payorId, string text) + { + return new Coverage + { + Status = FinancialResourceStatusCodes.Active, + Beneficiary = new ResourceReference($"Patient/{patientId}"), + Payor = { new ResourceReference($"Organization/{payorId}") }, + Class = + { + new Coverage.ClassComponent + { + Type = new CodeableConcept("http://terminology.hl7.org/CodeSystem/coverage-class", "plan"), + Value = "semantic-plan", + Name = text, + }, + }, + }; + } + + private async Task CreateAsync(T resource) + where T : Resource + { + using FhirResponse response = await _client.CreateAsync(resource); + return response.Resource; + } + } +} diff --git a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/StartupForSemanticSearchTests.cs b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/StartupForSemanticSearchTests.cs new file mode 100644 index 0000000000..442093179b --- /dev/null +++ b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Rest/StartupForSemanticSearchTests.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; + +namespace Microsoft.Health.Fhir.Tests.E2E.Rest +{ + [RequiresIsolatedDatabase] + public sealed class StartupForSemanticSearchTests : StartupBaseForCustomProviders + { + public StartupForSemanticSearchTests(IConfiguration configuration) + : base(ConfigureVectorSearch(configuration)) + { + } + + public override void ConfigureServices(IServiceCollection services) + { + base.ConfigureServices(services); + + services.Replace(ServiceDescriptor.Singleton(new SemanticSearchTestParameterResolver())); + services.Replace(ServiceDescriptor.Scoped(_ => new DeterministicEmbeddingClient())); + } + + private static IConfiguration ConfigureVectorSearch(IConfiguration configuration) + { + configuration["FhirServer:CoreFeatures:VectorSearch:Enabled"] = "true"; + configuration["FhirServer:CoreFeatures:VectorSearch:Embedding:Endpoint"] = "https://semantic-search.test"; + configuration["FhirServer:CoreFeatures:VectorSearch:Embedding:DeploymentName"] = "deterministic"; + configuration["FhirServer:CoreFeatures:VectorSearch:Embedding:ModelName"] = "deterministic"; + configuration["FhirServer:CoreFeatures:VectorSearch:Embedding:ModelVersion"] = "1"; + configuration["FhirServer:CoreFeatures:VectorSearch:Embedding:Dimensions"] = "1536"; + return configuration; + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/InProcTestFhirServer.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/InProcTestFhirServer.cs index cdbecc8ffc..f7d0453bc7 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/InProcTestFhirServer.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/InProcTestFhirServer.cs @@ -42,14 +42,14 @@ public InProcTestFhirServer(DataStore dataStore, Type startupType) : base(new Uri("http://localhost/")) { var projectDir = GetProjectPath("src", startupType); - var testConfigPath = Path.GetFullPath("testconfiguration.json"); + var testConfigPath = Path.Combine(AppContext.BaseDirectory, "testconfiguration.json"); var launchSettings = JObject.Parse(File.ReadAllText(Path.Combine(projectDir, "Properties", "launchSettings.json"))); var configuration = launchSettings["profiles"][dataStore.ToString()]["environmentVariables"].Cast().ToDictionary(p => p.Name, p => p.Value.ToString()); configuration["ASPNETCORE_FORWARDEDHEADERS_ENABLED"] = "true"; - configuration["TestAuthEnvironment:FilePath"] = "testauthenvironment.json"; + configuration["TestAuthEnvironment:FilePath"] = Path.Combine(AppContext.BaseDirectory, "testauthenvironment.json"); configuration["FhirServer:Security:Enabled"] = "true"; configuration["DevelopmentIdentityProvider:Enabled"] = "true"; @@ -90,6 +90,15 @@ public InProcTestFhirServer(DataStore dataStore, Type startupType) configuration["FhirServer:CoreFeatures:MaxIncludeCountPerSearch"] = "10"; configuration["FhirServer:CoreFeatures:DefaultIncludeCountPerSearch"] = "10"; + if (dataStore == DataStore.SqlServer) + { + string sqlConnectionString = Environment.GetEnvironmentVariable("SqlServer__ConnectionString"); + if (!string.IsNullOrWhiteSpace(sqlConnectionString)) + { + configuration["SqlServer:ConnectionString"] = sqlConnectionString; + } + } + if (startupType.IsDefined(typeof(RequiresIsolatedDatabaseAttribute))) { // Alter the configuration so that the server will create a new, isolated database/container. diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs index f35083cbf6..72463277fc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs @@ -47,6 +47,7 @@ using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Features.Search.SemanticSearch; using Microsoft.Health.Fhir.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Messages.Create; using Microsoft.Health.Fhir.Core.Messages.Delete; @@ -363,6 +364,9 @@ public async Task InitializeAsync() logger); var collection = new ServiceCollection(); + var semanticSearchEvidenceFilter = Substitute.For(); + semanticSearchEvidenceFilter.FilterAsync(Arg.Any(), Arg.Any()) + .Returns(callInfo => callInfo.Arg()); // Register request handlers collection.AddSingleton(typeof(IRequestHandler), new CreateResourceHandler(DataStore, new Lazy(() => ConformanceProvider), resourceWrapperFactory, _resourceIdProvider, new ResourceReferenceResolver(SearchService, new TestQueryStringParser(), Substitute.For>()), DisabledFhirAuthorizationService.Instance)); @@ -370,7 +374,7 @@ public async Task InitializeAsync() collection.AddSingleton(typeof(IRequestHandler), GetResourceHandler); collection.AddSingleton(typeof(IRequestHandler), new DeleteResourceHandler(DataStore, new Lazy(() => ConformanceProvider), resourceWrapperFactory, _resourceIdProvider, DisabledFhirAuthorizationService.Instance, deleter)); collection.AddSingleton(typeof(IRequestHandler), new SearchResourceHistoryHandler(SearchService, bundleFactory, DisabledFhirAuthorizationService.Instance, new DataResourceFilter(MissingDataFilterCriteria.Default))); - collection.AddSingleton(typeof(IRequestHandler), new SearchResourceHandler(SearchService, bundleFactory, DisabledFhirAuthorizationService.Instance, new DataResourceFilter(MissingDataFilterCriteria.Default))); + collection.AddSingleton(typeof(IRequestHandler), new SearchResourceHandler(SearchService, bundleFactory, DisabledFhirAuthorizationService.Instance, new DataResourceFilter(MissingDataFilterCriteria.Default), semanticSearchEvidenceFilter)); // Register pipeline behaviors for search parameter handling collection.AddTransient>( diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerTransactionScopeTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerTransactionScopeTests.cs index 38e8fe9fac..6cd2a8091b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerTransactionScopeTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerTransactionScopeTests.cs @@ -117,6 +117,334 @@ INSERT INTO Resource } } + [Fact] + public async Task GivenALinkedSource_WhenHardDeletingInsideAnAmbientTransaction_ThenRefreshIsEnqueuedWithoutChangingTheOwnerVersion() + { + string sourceResourceId = Guid.NewGuid().ToString(); + string ownerResourceId = Guid.NewGuid().ToString(); + string modelName = Guid.NewGuid().ToString(); + + using (SqlConnectionWrapper connectionWrapper = await _fixture.SqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(CancellationToken.None, false)) + using (SqlCommandWrapper command = connectionWrapper.CreateRetrySqlCommand()) + { + command.CommandText = @" +DECLARE @ResourceTypeId smallint = (SELECT TOP (1) ResourceTypeId FROM dbo.ResourceType ORDER BY ResourceTypeId) +DECLARE @SearchParamId smallint = (SELECT TOP (1) SearchParamId FROM dbo.SearchParam ORDER BY SearchParamId) +DECLARE @EmbeddingModelId smallint +DECLARE @BaseSurrogateId bigint = DATEDIFF_BIG(millisecond, CONVERT(datetime2, '0001-01-01'), SYSUTCDATETIME()) * CONVERT(bigint, 80000) +DECLARE @SourceHistorySurrogateId bigint = @BaseSurrogateId + NEXT VALUE FOR dbo.ResourceSurrogateIdUniquifierSequence +DECLARE @SourceCurrentSurrogateId bigint = @BaseSurrogateId + NEXT VALUE FOR dbo.ResourceSurrogateIdUniquifierSequence +DECLARE @OwnerSurrogateId bigint = @BaseSurrogateId + NEXT VALUE FOR dbo.ResourceSurrogateIdUniquifierSequence +DECLARE @InitialTranCount int = @@TRANCOUNT +DECLARE @HistoryDeleteTranCount int +DECLARE @SourceDeleteTranCount int +DECLARE @HistoryCount int +DECLARE @SourceCount int +DECLARE @OwnerVersion int +DECLARE @OwnerVectorCount int +DECLARE @RefreshJobCount int + +BEGIN TRY + BEGIN TRANSACTION + + INSERT INTO dbo.EmbeddingModel (ModelName, ModelVersion, Dimension) + VALUES (@ModelName, 'test', 1536) + SET @EmbeddingModelId = CONVERT(smallint, SCOPE_IDENTITY()) + + INSERT INTO dbo.Resource + (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash) + VALUES + (@ResourceTypeId, @SourceResourceId, 1, 1, @SourceHistorySurrogateId, 0, 'PUT', 0x01, 0, NULL), + (@ResourceTypeId, @SourceResourceId, 2, 0, @SourceCurrentSurrogateId, 0, 'PUT', 0x01, 0, NULL), + (@ResourceTypeId, @OwnerResourceId, 7, 0, @OwnerSurrogateId, 0, 'PUT', 0x01, 0, NULL) + + INSERT INTO dbo.VectorSearchParam + (ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, Embedding) + VALUES + (@ResourceTypeId, @OwnerSurrogateId, @SearchParamId, 0, @EmbeddingModelId, N'source text', HASHBYTES('SHA2_256', N'source text'), @ResourceTypeId, @SourceResourceId, '2', N'content', CAST(CONCAT('[', REPLICATE('0,', 1535), '0]') AS vector(1536))) + + EXECUTE dbo.HardDeleteResource + @ResourceTypeId = @ResourceTypeId, + @ResourceId = @SourceResourceId, + @KeepCurrentVersion = 1, + @IsResourceChangeCaptureEnabled = 0 + + SET @HistoryDeleteTranCount = @@TRANCOUNT + SET @HistoryCount = (SELECT COUNT(*) FROM dbo.Resource WHERE ResourceTypeId = @ResourceTypeId AND ResourceId = @SourceResourceId AND IsHistory = 1) + + EXECUTE dbo.HardDeleteResourceWithVectorSearchSourceRefresh + @ResourceTypeId = @ResourceTypeId, + @ResourceId = @SourceResourceId, + @KeepCurrentVersion = 0, + @IsResourceChangeCaptureEnabled = 0 + + SET @SourceDeleteTranCount = @@TRANCOUNT + SET @SourceCount = (SELECT COUNT(*) FROM dbo.Resource WHERE ResourceTypeId = @ResourceTypeId AND ResourceId = @SourceResourceId) + SET @OwnerVersion = (SELECT Version FROM dbo.Resource WHERE ResourceTypeId = @ResourceTypeId AND ResourceId = @OwnerResourceId AND IsHistory = 0) + SET @OwnerVectorCount = (SELECT COUNT(*) FROM dbo.VectorSearchParam WHERE ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId = @OwnerSurrogateId AND SourceResourceTypeId = @ResourceTypeId AND SourceResourceId = @SourceResourceId) + SET @RefreshJobCount = + ( + SELECT COUNT(*) + FROM dbo.JobQueue + WHERE QueueType = 6 + AND JSON_VALUE(Definition, '$.TypeId') = '11' + AND JSON_VALUE(Definition, '$.SourceResourceType') COLLATE Latin1_General_100_CS_AS = (SELECT Name FROM dbo.ResourceType WHERE ResourceTypeId = @ResourceTypeId) + AND JSON_VALUE(Definition, '$.SourceResourceId') = @SourceResourceId + AND JSON_VALUE(Definition, '$.SourceResourceVersion') = '3' + ) + + ROLLBACK TRANSACTION +END TRY +BEGIN CATCH + IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION + THROW +END CATCH + +SELECT @InitialTranCount, + @HistoryDeleteTranCount, + @SourceDeleteTranCount, + @HistoryCount, + @SourceCount, + @OwnerVersion, + @OwnerVectorCount, + @RefreshJobCount"; + + command.Parameters.Add(new SqlParameter { ParameterName = "SourceResourceId", Value = sourceResourceId }); + command.Parameters.Add(new SqlParameter { ParameterName = "OwnerResourceId", Value = ownerResourceId }); + command.Parameters.Add(new SqlParameter { ParameterName = "ModelName", Value = modelName }); + + using (SqlDataReader reader = await command.ExecuteReaderAsync(CancellationToken.None)) + { + Assert.True(await reader.ReadAsync(CancellationToken.None)); + Assert.Equal(0, reader.GetInt32(0)); + Assert.Equal(1, reader.GetInt32(1)); + Assert.Equal(1, reader.GetInt32(2)); + Assert.Equal(0, reader.GetInt32(3)); + Assert.Equal(0, reader.GetInt32(4)); + Assert.Equal(7, reader.GetInt32(5)); + Assert.Equal(1, reader.GetInt32(6)); + Assert.Equal(1, reader.GetInt32(7)); + } + } + } + + [Fact] + public async Task GivenALinkedSourceUpdate_WhenMergingWithSourceRefresh_ThenRefreshIsEnqueuedForTheNewVersion() + { + string sourceResourceId = Guid.NewGuid().ToString(); + string ownerResourceId = Guid.NewGuid().ToString(); + string modelName = Guid.NewGuid().ToString(); + + using (SqlConnectionWrapper connectionWrapper = await _fixture.SqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(CancellationToken.None, false)) + using (SqlCommandWrapper command = connectionWrapper.CreateRetrySqlCommand()) + { + command.CommandText = @" +DECLARE @ResourceTypeId smallint = (SELECT TOP (1) ResourceTypeId FROM dbo.ResourceType ORDER BY ResourceTypeId) +DECLARE @SearchParamId smallint = (SELECT TOP (1) SearchParamId FROM dbo.SearchParam ORDER BY SearchParamId) +DECLARE @EmbeddingModelId smallint +DECLARE @BaseSurrogateId bigint = DATEDIFF_BIG(millisecond, CONVERT(datetime2, '0001-01-01'), SYSUTCDATETIME()) * CONVERT(bigint, 80000) +DECLARE @SourceV1SurrogateId bigint = @BaseSurrogateId + NEXT VALUE FOR dbo.ResourceSurrogateIdUniquifierSequence +DECLARE @SourceV2SurrogateId bigint = @BaseSurrogateId + NEXT VALUE FOR dbo.ResourceSurrogateIdUniquifierSequence +DECLARE @OwnerSurrogateId bigint = @BaseSurrogateId + NEXT VALUE FOR dbo.ResourceSurrogateIdUniquifierSequence +DECLARE @InitialTranCount int = @@TRANCOUNT +DECLARE @MergeTranCount int +DECLARE @SourceVersion int +DECLARE @RefreshJobCount int +DECLARE @Resources dbo.ResourceList +DECLARE @ResourceWriteClaims dbo.ResourceWriteClaimList +DECLARE @ReferenceSearchParams dbo.ReferenceSearchParamList +DECLARE @TokenSearchParams dbo.TokenSearchParamList +DECLARE @TokenTexts dbo.TokenTextList +DECLARE @StringSearchParams dbo.StringSearchParamList +DECLARE @UriSearchParams dbo.UriSearchParamList +DECLARE @NumberSearchParams dbo.NumberSearchParamList +DECLARE @QuantitySearchParams dbo.QuantitySearchParamList +DECLARE @DateTimeSearchParms dbo.DateTimeSearchParamList +DECLARE @VectorSearchParams dbo.VectorSearchParamList +DECLARE @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList +DECLARE @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList +DECLARE @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList +DECLARE @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList +DECLARE @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList +DECLARE @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList + +BEGIN TRY + BEGIN TRANSACTION + + INSERT INTO dbo.EmbeddingModel (ModelName, ModelVersion, Dimension) + VALUES (@ModelName, 'test', 1536) + SET @EmbeddingModelId = CONVERT(smallint, SCOPE_IDENTITY()) + + INSERT INTO dbo.Resource + (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash) + VALUES + (@ResourceTypeId, @SourceResourceId, 1, 0, @SourceV1SurrogateId, 0, 'PUT', 0x01, 0, NULL), + (@ResourceTypeId, @OwnerResourceId, 1, 0, @OwnerSurrogateId, 0, 'PUT', 0x01, 0, NULL) + + INSERT INTO dbo.VectorSearchParam + (ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, Embedding) + VALUES + (@ResourceTypeId, @OwnerSurrogateId, @SearchParamId, 0, @EmbeddingModelId, N'source text', HASHBYTES('SHA2_256', N'source text'), @ResourceTypeId, @SourceResourceId, '1', N'content', CAST(CONCAT('[', REPLICATE('0,', 1535), '0]') AS vector(1536))) + + INSERT INTO @Resources + (ResourceTypeId, ResourceSurrogateId, ResourceId, Version, HasVersionToCompare, IsDeleted, IsHistory, KeepHistory, RawResource, IsRawResourceMetaSet, RequestMethod, SearchParamHash) + VALUES + (@ResourceTypeId, @SourceV2SurrogateId, @SourceResourceId, 2, 1, 0, 0, 1, 0x02, 0, 'PUT', NULL) + + EXECUTE dbo.MergeResourcesWithVectorSearchSourceRefresh + @Resources = @Resources, + @ResourceWriteClaims = @ResourceWriteClaims, + @ReferenceSearchParams = @ReferenceSearchParams, + @TokenSearchParams = @TokenSearchParams, + @TokenTexts = @TokenTexts, + @StringSearchParams = @StringSearchParams, + @UriSearchParams = @UriSearchParams, + @NumberSearchParams = @NumberSearchParams, + @QuantitySearchParams = @QuantitySearchParams, + @DateTimeSearchParms = @DateTimeSearchParms, + @VectorSearchParams = @VectorSearchParams, + @ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams, + @TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams, + @TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams, + @TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams, + @TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams, + @TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams + + SET @MergeTranCount = @@TRANCOUNT + SET @SourceVersion = (SELECT Version FROM dbo.Resource WHERE ResourceTypeId = @ResourceTypeId AND ResourceId = @SourceResourceId AND IsHistory = 0) + SET @RefreshJobCount = + ( + SELECT COUNT(*) + FROM dbo.JobQueue + WHERE QueueType = 6 + AND JSON_VALUE(Definition, '$.TypeId') = '11' + AND JSON_VALUE(Definition, '$.SourceResourceType') COLLATE Latin1_General_100_CS_AS = (SELECT Name FROM dbo.ResourceType WHERE ResourceTypeId = @ResourceTypeId) + AND JSON_VALUE(Definition, '$.SourceResourceId') = @SourceResourceId + AND JSON_VALUE(Definition, '$.SourceResourceVersion') = '2' + ) + + ROLLBACK TRANSACTION +END TRY +BEGIN CATCH + IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION + THROW +END CATCH + +SELECT @InitialTranCount, + @MergeTranCount, + @SourceVersion, + @RefreshJobCount"; + + command.Parameters.Add(new SqlParameter { ParameterName = "SourceResourceId", Value = sourceResourceId }); + command.Parameters.Add(new SqlParameter { ParameterName = "OwnerResourceId", Value = ownerResourceId }); + command.Parameters.Add(new SqlParameter { ParameterName = "ModelName", Value = modelName }); + + using (SqlDataReader reader = await command.ExecuteReaderAsync(CancellationToken.None)) + { + Assert.True(await reader.ReadAsync(CancellationToken.None)); + Assert.Equal(0, reader.GetInt32(0)); + Assert.Equal(1, reader.GetInt32(1)); + Assert.Equal(2, reader.GetInt32(2)); + Assert.Equal(1, reader.GetInt32(3)); + } + } + } + + [Fact] + public async Task GivenResourceVersionsWithOwnedVectors_WhenHardDeletingInsideAnAmbientTransaction_ThenDeletedOwnedVectorsAreRemoved() + { + string resourceId = Guid.NewGuid().ToString(); + string modelName = Guid.NewGuid().ToString(); + + using (SqlConnectionWrapper connectionWrapper = await _fixture.SqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(CancellationToken.None, false)) + using (SqlCommandWrapper command = connectionWrapper.CreateRetrySqlCommand()) + { + command.CommandText = @" +DECLARE @ResourceTypeId smallint = (SELECT TOP (1) ResourceTypeId FROM dbo.ResourceType ORDER BY ResourceTypeId) +DECLARE @SearchParamId smallint = (SELECT TOP (1) SearchParamId FROM dbo.SearchParam ORDER BY SearchParamId) +DECLARE @EmbeddingModelId smallint +DECLARE @BaseSurrogateId bigint = DATEDIFF_BIG(millisecond, CONVERT(datetime2, '0001-01-01'), SYSUTCDATETIME()) * CONVERT(bigint, 80000) +DECLARE @HistorySurrogateId bigint = @BaseSurrogateId + NEXT VALUE FOR dbo.ResourceSurrogateIdUniquifierSequence +DECLARE @CurrentSurrogateId bigint = @BaseSurrogateId + NEXT VALUE FOR dbo.ResourceSurrogateIdUniquifierSequence +DECLARE @InitialTranCount int = @@TRANCOUNT +DECLARE @HistoryDeleteTranCount int +DECLARE @DeleteTranCount int +DECLARE @ResourceCountAfterHistoryDelete int +DECLARE @VectorCountAfterHistoryDelete int +DECLARE @ResourceCount int +DECLARE @VectorCount int + +BEGIN TRY + BEGIN TRANSACTION + + INSERT INTO dbo.EmbeddingModel (ModelName, ModelVersion, Dimension) + VALUES (@ModelName, 'test', 1536) + SET @EmbeddingModelId = CONVERT(smallint, SCOPE_IDENTITY()) + + INSERT INTO dbo.Resource + (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash) + VALUES + (@ResourceTypeId, @ResourceId, 1, 1, @HistorySurrogateId, 0, 'PUT', 0x01, 0, NULL), + (@ResourceTypeId, @ResourceId, 2, 0, @CurrentSurrogateId, 0, 'PUT', 0x01, 0, NULL) + + INSERT INTO dbo.VectorSearchParam + (ResourceTypeId, ResourceSurrogateId, SearchParamId, ChunkOrdinal, EmbeddingModelId, ChunkText, SourceTextHash, SourceResourceTypeId, SourceResourceId, SourceResourceVersion, SourcePath, Embedding) + VALUES + (@ResourceTypeId, @HistorySurrogateId, @SearchParamId, 0, @EmbeddingModelId, N'history text', HASHBYTES('SHA2_256', N'history text'), @ResourceTypeId, @ResourceId, '1', N'content', CAST(CONCAT('[', REPLICATE('0,', 1535), '0]') AS vector(1536))), + (@ResourceTypeId, @CurrentSurrogateId, @SearchParamId, 0, @EmbeddingModelId, N'current text', HASHBYTES('SHA2_256', N'current text'), @ResourceTypeId, @ResourceId, '2', N'content', CAST(CONCAT('[', REPLICATE('0,', 1535), '0]') AS vector(1536))) + + EXECUTE dbo.HardDeleteResource + @ResourceTypeId = @ResourceTypeId, + @ResourceId = @ResourceId, + @KeepCurrentVersion = 1, + @IsResourceChangeCaptureEnabled = 0 + + SET @HistoryDeleteTranCount = @@TRANCOUNT + SET @ResourceCountAfterHistoryDelete = (SELECT COUNT(*) FROM dbo.Resource WHERE ResourceTypeId = @ResourceTypeId AND ResourceId = @ResourceId) + SET @VectorCountAfterHistoryDelete = (SELECT COUNT(*) FROM dbo.VectorSearchParam WHERE ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId IN (@HistorySurrogateId, @CurrentSurrogateId)) + + EXECUTE dbo.HardDeleteResource + @ResourceTypeId = @ResourceTypeId, + @ResourceId = @ResourceId, + @KeepCurrentVersion = 0, + @IsResourceChangeCaptureEnabled = 0 + + SET @DeleteTranCount = @@TRANCOUNT + SET @ResourceCount = (SELECT COUNT(*) FROM dbo.Resource WHERE ResourceTypeId = @ResourceTypeId AND ResourceId = @ResourceId) + SET @VectorCount = (SELECT COUNT(*) FROM dbo.VectorSearchParam WHERE ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId IN (@HistorySurrogateId, @CurrentSurrogateId)) + + ROLLBACK TRANSACTION +END TRY +BEGIN CATCH + IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION + THROW +END CATCH + +SELECT @InitialTranCount, + @HistoryDeleteTranCount, + @DeleteTranCount, + @ResourceCountAfterHistoryDelete, + @VectorCountAfterHistoryDelete, + @ResourceCount, + @VectorCount"; + + command.Parameters.Add(new SqlParameter { ParameterName = "ResourceId", Value = resourceId }); + command.Parameters.Add(new SqlParameter { ParameterName = "ModelName", Value = modelName }); + + using (SqlDataReader reader = await command.ExecuteReaderAsync(CancellationToken.None)) + { + Assert.True(await reader.ReadAsync(CancellationToken.None)); + Assert.Equal(0, reader.GetInt32(0)); + Assert.Equal(1, reader.GetInt32(1)); + Assert.Equal(1, reader.GetInt32(2)); + Assert.Equal(1, reader.GetInt32(3)); + Assert.Equal(1, reader.GetInt32(4)); + Assert.Equal(0, reader.GetInt32(5)); + Assert.Equal(0, reader.GetInt32(6)); + } + } + } + private static async Task VerifyCommandResults(SqlConnectionWrapper connectionWrapper, string newId, bool shouldFind, string tableHints = "") { using (SqlCommandWrapper sqlCommandWrapper = connectionWrapper.CreateRetrySqlCommand()) diff --git a/tools/PerfTester/Program.cs b/tools/PerfTester/Program.cs index fa49b8f4ce..2ea0964cf1 100644 --- a/tools/PerfTester/Program.cs +++ b/tools/PerfTester/Program.cs @@ -120,7 +120,6 @@ public static void Main() SwitchToResourceView(); ExecuteParallelCalls(resourceIds); resourceIds = GetRandomIds(); - ExecuteParallelCalls(resourceIds); // compare this SwitchToResourceTable(); ExecuteParallelCalls(resourceIds); } @@ -484,13 +483,13 @@ private static void ExecuteParallelCalls(ReadOnlyList<(short ResourceTypeId, str { var typeId = resourceIds.Item2.First().ResourceTypeId; var id = resourceIds.Item2.First().ResourceId; - _store.HardDeleteAsync(typeId, id, false, false, CancellationToken.None).Wait(); + _store.HardDeleteAsync(typeId, id, false, false, false, CancellationToken.None).Wait(); } else if (_callType == "HardDeleteWithInvisible") { var typeId = resourceIds.Item2.First().ResourceTypeId; var id = resourceIds.Item2.First().ResourceId; - _store.HardDeleteAsync(typeId, id, false, true, CancellationToken.None).Wait(); + _store.HardDeleteAsync(typeId, id, false, true, false, CancellationToken.None).Wait(); } else {