diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs index 3c2cee7b2d..fd37b26671 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs @@ -10,8 +10,8 @@ using System.Net.Http; using System.Reflection; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; -using System.Threading.Tasks; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; using Microsoft.Azure.Cosmos; @@ -88,6 +88,48 @@ public CosmosFhirDataStoreTests() requestContextAccessor); } + [Fact] + public void GivenCanonicallyEquivalentJapaneseText_WhenComparingResourcesWithoutVersionMetadata_ThenTheDataMatches() + { + const string existingData = "{\"resourceType\":\"Patient\",\"id\":\"id1\",\"meta\":{\"versionId\":\"1\",\"lastUpdated\":\"2026-08-26T21:00:00+00:00\"},\"name\":[{\"text\":\"\u304C\"}]}"; + const string incomingData = "{\"resourceType\":\"Patient\",\"id\":\"id1\",\"meta\":{\"versionId\":\"2\",\"lastUpdated\":\"2026-08-26T21:01:00+00:00\"},\"name\":[{\"text\":\"\u304B\u3099\"}]}"; + + var lastModified = new DateTimeOffset(2026, 8, 26, 21, 0, 0, TimeSpan.Zero); + var request = new ResourceRequest(HttpMethod.Put, "http://fhir"); + + var existingResource = new FhirCosmosResourceWrapper( + "id1", + "1", + "Patient", + new RawResource(existingData, FhirResourceFormat.Json, isMetaSet: true), + request, + lastModified, + deleted: false, + history: false, + searchIndices: null, + compartmentIndices: null, + lastModifiedClaims: null); + + var incomingResource = new FhirCosmosResourceWrapper( + "id1", + "2", + "Patient", + new RawResource(incomingData, FhirResourceFormat.Json, isMetaSet: true), + request, + lastModified.AddMinutes(1), + deleted: false, + history: false, + searchIndices: null, + compartmentIndices: null, + lastModifiedClaims: null); + + var existingResourceData = CosmosFhirDataStore.RemoveVersionIdAndLastUpdatedFromMeta(existingResource); + var incomingResourceData = CosmosFhirDataStore.RemoveVersionIdAndLastUpdatedFromMeta(incomingResource); + + Assert.Equal(existingResourceData, incomingResourceData); + Assert.True(existingResourceData.IsNormalized(NormalizationForm.FormC)); + } + [Fact] public async Task GivenAQuery_WhenASinglePageReturnsRequestedCount_ASingleQueryIsPerformced() { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs index d0c2b72a0c..23a8183b77 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs @@ -12,22 +12,20 @@ using System.Linq; using System.Net; using System.Text; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using EnsureThat; +using Hl7.Fhir.Utility; using Microsoft.Azure.Cosmos; using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Health.Abstractions.Exceptions; -using Microsoft.Health.Core; using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; -using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Conformance; using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Definition; @@ -432,10 +430,13 @@ await retryPolicy.ExecuteAsync( // If not a delete then check if its an update with no data change if (!cosmosWrapper.IsDeleted) { - // check if the new resource data is same as existing resource data - if (string.Equals(RemoveVersionIdAndLastUpdatedFromMeta(existingItemResource), RemoveVersionIdAndLastUpdatedFromMeta(cosmosWrapper), StringComparison.Ordinal)) + string existingData = RemoveVersionIdAndLastUpdatedFromMeta(existingItemResource); + string newData = RemoveVersionIdAndLastUpdatedFromMeta(cosmosWrapper); + + // Check if the new resource data is same as existing resource data + if (string.Equals(existingData, newData, StringComparison.Ordinal)) { - // Do not store the duplicate data, for a update with no impact - returning existingItemResource as no updates + // Do not store the duplicate data, for a update with no impact - returning existingItemResource as no updates. return new UpsertOutcome(existingItemResource, SaveOutcomeType.Updated); } } @@ -876,11 +877,19 @@ private static string RemoveTrailingZerosFromMillisecondsForAGivenDate(DateTimeO return formattedDate.Replace(milliseconds, trimmedMilliseconds, StringComparison.Ordinal); } - private static string RemoveVersionIdAndLastUpdatedFromMeta(FhirCosmosResourceWrapper resourceWrapper) + internal static string RemoveVersionIdAndLastUpdatedFromMeta(FhirCosmosResourceWrapper resourceWrapper) { var versionToReplace = resourceWrapper.RawResource.IsMetaSet ? resourceWrapper.Version : "1"; - var rawResource = resourceWrapper.RawResource.Data.Replace($"\"versionId\":\"{versionToReplace}\"", string.Empty, StringComparison.Ordinal); - return rawResource.Replace($"\"lastUpdated\":\"{RemoveTrailingZerosFromMillisecondsForAGivenDate(resourceWrapper.LastModified)}\"", string.Empty, StringComparison.Ordinal); + var rawResource = resourceWrapper.RawResource.Data + .Replace($"\"versionId\":\"{versionToReplace}\"", string.Empty, StringComparison.Ordinal) + .Replace($"\"lastUpdated\":\"{RemoveTrailingZerosFromMillisecondsForAGivenDate(resourceWrapper.LastModified)}\"", string.Empty, StringComparison.Ordinal); + + // StringComparison.Ordinal performs a byte-by-byte comparison without any Unicode normalization. This is problematic for Japanese characters because: Japanese characters + // can be represented in multiple equivalent Unicode forms: NFD (Canonical Decomposition) or NFC (Canonical Composition). + // If a client sends Japanese text in one normalization form(e.g., NFD) but a previous version stored it in another form(e.g., NFC), the Ordinal comparison will fail even though + // the characters look identical. + // English/ASCII characters don't have normalization variants, so this only affects languages like Japanese, Korean, Vietnamese, etc. + return rawResource.Normalize(NormalizationForm.FormC); } public async Task BuildAsync(ICapabilityStatementBuilder builder, CancellationToken cancellationToken)