From b2a611f81e18d8d86c245d3ed17ec60fa9349729 Mon Sep 17 00:00:00 2001 From: uzer_007 Date: Sat, 18 Jul 2026 22:39:50 +0300 Subject: [PATCH 1/2] Fix UTF-8 partial match ranking Calculate partial-match penalties from unmatched Unicode characters and cache character counts for indexed words. --- cpp_src/core/ft/ft_fast/dataholder.cc | 4 ++++ cpp_src/core/ft/ft_fast/dataholder.h | 16 ++++++++++++++++ cpp_src/core/ft/ft_fast/dataprocessor.cc | 6 +++--- cpp_src/core/ft/ft_fast/selecterimpl.h | 10 ++++++---- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/cpp_src/core/ft/ft_fast/dataholder.cc b/cpp_src/core/ft/ft_fast/dataholder.cc index bbeb80623..8b0a45e1e 100644 --- a/cpp_src/core/ft/ft_fast/dataholder.cc +++ b/cpp_src/core/ft/ft_fast/dataholder.cc @@ -94,6 +94,7 @@ size_t DataHolder::GetMemStat() { for (auto& w : words_) { res += sizeof(w) + w.vids.heap_size(); } + res += wordsCharsLen_.capacity() * sizeof(wordsCharsLen_[0]); return res; } @@ -101,6 +102,7 @@ template void DataHolder::Clear() { IDataHolder::Clear(); words_.resize(0); + wordsCharsLen_.resize(0); } template @@ -110,10 +112,12 @@ void DataHolder::StartCommit(bool complete_updated) { Clear(); words_.clear(); + wordsCharsLen_.clear(); lastStepWords_.clear(); } else if (NeedRecommitLast()) { status_ = RecommitLast; words_.erase(words_.begin() + steps.back().wordOffset_, words_.end()); + wordsCharsLen_.erase(wordsCharsLen_.begin() + steps.back().wordOffset_, wordsCharsLen_.end()); for (auto& word : words_) { word.RestoreState(); diff --git a/cpp_src/core/ft/ft_fast/dataholder.h b/cpp_src/core/ft/ft_fast/dataholder.h index e009b0530..43ab49e3d 100644 --- a/cpp_src/core/ft/ft_fast/dataholder.h +++ b/cpp_src/core/ft/ft_fast/dataholder.h @@ -185,6 +185,8 @@ class [[nodiscard]] IDataHolder { template class [[nodiscard]] DataHolder : public IDataHolder { public: + using WordCharsLenType = uint16_t; + explicit DataHolder(FTConfig* c); void Process(VDocsTexts& vdocsTexts, const std::vector& vdocsIds, size_t numDocsTotal, size_t fieldSize, bool multithread, std::vector>& wordsCounts) final; @@ -193,6 +195,19 @@ class [[nodiscard]] DataHolder : public IDataHolder { void Clear() override final; std::vector>& GetWords() noexcept { return words_; } const std::vector>& GetWords() const noexcept { return words_; } + void ReserveWords(size_t capacity) { + words_.reserve(capacity); + wordsCharsLen_.reserve(capacity); + } + void AddWordCharsLen(WordCharsLenType charsLen) { + assertrx_dbg(wordsCharsLen_.size() < words_.size()); + wordsCharsLen_.emplace_back(charsLen); + } + WordCharsLenType GetWordCharsLen(WordIdType id) const noexcept { + assertrx(!id.IsEmpty()); + assertrx(id.b.id < wordsCharsLen_.size()); + return wordsCharsLen_[id.b.id]; + } PackedWordEntry& GetWordEntry(WordIdType id) noexcept { assertrx(!id.IsEmpty()); assertrx(id.b.id < words_.size()); @@ -204,6 +219,7 @@ class [[nodiscard]] DataHolder : public IDataHolder { return words_[id.b.id]; } std::vector> words_; + std::vector wordsCharsLen_; }; } // namespace reindexer diff --git a/cpp_src/core/ft/ft_fast/dataprocessor.cc b/cpp_src/core/ft/ft_fast/dataprocessor.cc index 142c295b1..7545cd52c 100644 --- a/cpp_src/core/ft/ft_fast/dataprocessor.cc +++ b/cpp_src/core/ft/ft_fast/dataprocessor.cc @@ -80,9 +80,7 @@ typename DataProcessor::WordsVector DataProcessor::insertIntoSuf word_hash wh; suffix.reserve(words_um.size() * 20, words_um.size()); - if (words.empty()) { - words.reserve(words_um.size()); - } + holder.ReserveWords(words.size() + words_um.size()); WordsVector found; found.reserve(words_um.size()); @@ -99,7 +97,9 @@ typename DataProcessor::WordsVector DataProcessor::insertIntoSuf } found.emplace_back(keyIt.first); + const auto charsLen = static_cast::WordCharsLenType>(getUTF8StringCharactersCount(keyIt.first)); words.emplace_back(); + holder.AddWordCharsLen(charsLen); pos = holder.BuildWordId(id); suffix.insert(keyIt.first, pos); holder.lastStepWords_[wh(keyIt.first)].emplace_back(pos); diff --git a/cpp_src/core/ft/ft_fast/selecterimpl.h b/cpp_src/core/ft/ft_fast/selecterimpl.h index 09a88b5d4..d1d288a6a 100644 --- a/cpp_src/core/ft/ft_fast/selecterimpl.h +++ b/cpp_src/core/ft/ft_fast/selecterimpl.h @@ -293,6 +293,7 @@ ft::TermResults Selector::buildTermResults(const FtDSLEntry& ter size_t matched = 0, vids = 0, excludedCnt = 0; const std::string& patternUtf8 = variant.PatternUtf8(); const size_t patternBytes = patternUtf8.length(); + const auto patternChars = static_cast(getUTF8StringCharactersCount(patternUtf8)); for (const auto& step : holder_.steps) { if (variant.lowRelevance && totalVids >= lowRelevanceLimit) { break; @@ -329,11 +330,12 @@ ft::TermResults Selector::buildTermResults(const FtDSLEntry& ter break; } - // ToDo fix it (broken for russian utf8 symbols) - const int matchDif = std::abs(long(word.length() - variant.PatternUtf8().length() + wordLengthBeforePattern)); + const uint32_t wordChars = holder_.GetWordCharsLen(wordId); + const uint32_t unmatchedChars = wordChars > patternChars ? wordChars - patternChars : 0; const float boost = std::max(getTermBoost(std::string(word)), variant.boost); - const float decreasePenalty = static_cast(holder_.cfg_->partialMatchDecrease * matchDif) / - std::max(variant.PatternUtf8().length(), kMinPartialMatchDenominator); + const float decreasePenalty = + (static_cast(holder_.cfg_->partialMatchDecrease) * static_cast(unmatchedChars)) / + static_cast(std::max(patternChars, kMinPartialMatchDenominator)); float proc = std::max(variant.proc - decreasePenalty, isPrefix ? rankingCfg.PrefixMin() : rankingCfg.SuffixMin()); proc = std::min(proc, variant.proc); if (boost > 0.0f) { From c33995fea90f688b4df579469e17cccf9b81a10a Mon Sep 17 00:00:00 2001 From: uzer_007 Date: Thu, 23 Jul 2026 23:58:34 +0300 Subject: [PATCH 2/2] Refine UTF-8 partial-match ranking implementation --- cpp_src/core/ft/ft_fast/dataholder.cc | 4 -- cpp_src/core/ft/ft_fast/dataholder.h | 16 ----- cpp_src/core/ft/ft_fast/dataprocessor.cc | 6 +- cpp_src/core/ft/ft_fast/selecterimpl.h | 2 +- cpp_src/estl/suffix_map.h | 5 +- cpp_src/gtests/tests/unit/ft/ft_generic.cc | 69 ++++++++++++++++++++++ 6 files changed, 76 insertions(+), 26 deletions(-) diff --git a/cpp_src/core/ft/ft_fast/dataholder.cc b/cpp_src/core/ft/ft_fast/dataholder.cc index 8b0a45e1e..bbeb80623 100644 --- a/cpp_src/core/ft/ft_fast/dataholder.cc +++ b/cpp_src/core/ft/ft_fast/dataholder.cc @@ -94,7 +94,6 @@ size_t DataHolder::GetMemStat() { for (auto& w : words_) { res += sizeof(w) + w.vids.heap_size(); } - res += wordsCharsLen_.capacity() * sizeof(wordsCharsLen_[0]); return res; } @@ -102,7 +101,6 @@ template void DataHolder::Clear() { IDataHolder::Clear(); words_.resize(0); - wordsCharsLen_.resize(0); } template @@ -112,12 +110,10 @@ void DataHolder::StartCommit(bool complete_updated) { Clear(); words_.clear(); - wordsCharsLen_.clear(); lastStepWords_.clear(); } else if (NeedRecommitLast()) { status_ = RecommitLast; words_.erase(words_.begin() + steps.back().wordOffset_, words_.end()); - wordsCharsLen_.erase(wordsCharsLen_.begin() + steps.back().wordOffset_, wordsCharsLen_.end()); for (auto& word : words_) { word.RestoreState(); diff --git a/cpp_src/core/ft/ft_fast/dataholder.h b/cpp_src/core/ft/ft_fast/dataholder.h index 43ab49e3d..e009b0530 100644 --- a/cpp_src/core/ft/ft_fast/dataholder.h +++ b/cpp_src/core/ft/ft_fast/dataholder.h @@ -185,8 +185,6 @@ class [[nodiscard]] IDataHolder { template class [[nodiscard]] DataHolder : public IDataHolder { public: - using WordCharsLenType = uint16_t; - explicit DataHolder(FTConfig* c); void Process(VDocsTexts& vdocsTexts, const std::vector& vdocsIds, size_t numDocsTotal, size_t fieldSize, bool multithread, std::vector>& wordsCounts) final; @@ -195,19 +193,6 @@ class [[nodiscard]] DataHolder : public IDataHolder { void Clear() override final; std::vector>& GetWords() noexcept { return words_; } const std::vector>& GetWords() const noexcept { return words_; } - void ReserveWords(size_t capacity) { - words_.reserve(capacity); - wordsCharsLen_.reserve(capacity); - } - void AddWordCharsLen(WordCharsLenType charsLen) { - assertrx_dbg(wordsCharsLen_.size() < words_.size()); - wordsCharsLen_.emplace_back(charsLen); - } - WordCharsLenType GetWordCharsLen(WordIdType id) const noexcept { - assertrx(!id.IsEmpty()); - assertrx(id.b.id < wordsCharsLen_.size()); - return wordsCharsLen_[id.b.id]; - } PackedWordEntry& GetWordEntry(WordIdType id) noexcept { assertrx(!id.IsEmpty()); assertrx(id.b.id < words_.size()); @@ -219,7 +204,6 @@ class [[nodiscard]] DataHolder : public IDataHolder { return words_[id.b.id]; } std::vector> words_; - std::vector wordsCharsLen_; }; } // namespace reindexer diff --git a/cpp_src/core/ft/ft_fast/dataprocessor.cc b/cpp_src/core/ft/ft_fast/dataprocessor.cc index 7545cd52c..142c295b1 100644 --- a/cpp_src/core/ft/ft_fast/dataprocessor.cc +++ b/cpp_src/core/ft/ft_fast/dataprocessor.cc @@ -80,7 +80,9 @@ typename DataProcessor::WordsVector DataProcessor::insertIntoSuf word_hash wh; suffix.reserve(words_um.size() * 20, words_um.size()); - holder.ReserveWords(words.size() + words_um.size()); + if (words.empty()) { + words.reserve(words_um.size()); + } WordsVector found; found.reserve(words_um.size()); @@ -97,9 +99,7 @@ typename DataProcessor::WordsVector DataProcessor::insertIntoSuf } found.emplace_back(keyIt.first); - const auto charsLen = static_cast::WordCharsLenType>(getUTF8StringCharactersCount(keyIt.first)); words.emplace_back(); - holder.AddWordCharsLen(charsLen); pos = holder.BuildWordId(id); suffix.insert(keyIt.first, pos); holder.lastStepWords_[wh(keyIt.first)].emplace_back(pos); diff --git a/cpp_src/core/ft/ft_fast/selecterimpl.h b/cpp_src/core/ft/ft_fast/selecterimpl.h index d1d288a6a..aed103664 100644 --- a/cpp_src/core/ft/ft_fast/selecterimpl.h +++ b/cpp_src/core/ft/ft_fast/selecterimpl.h @@ -330,7 +330,7 @@ ft::TermResults Selector::buildTermResults(const FtDSLEntry& ter break; } - const uint32_t wordChars = holder_.GetWordCharsLen(wordId); + const auto wordChars = static_cast(getUTF8StringCharactersCount(word)); const uint32_t unmatchedChars = wordChars > patternChars ? wordChars - patternChars : 0; const float boost = std::max(getTermBoost(std::string(word)), variant.boost); const float decreasePenalty = diff --git a/cpp_src/estl/suffix_map.h b/cpp_src/estl/suffix_map.h index 483dadd13..57435f9ab 100644 --- a/cpp_src/estl/suffix_map.h +++ b/cpp_src/estl/suffix_map.h @@ -18,6 +18,7 @@ class [[nodiscard]] suffix_map { private: typedef size_t size_type; typedef unsigned char char_type; + using WordsLenContainer = std::vector; class [[nodiscard]] value_type : public std::pair { public: @@ -194,7 +195,7 @@ class [[nodiscard]] suffix_map { size_t heap_size() noexcept { return (sa_.capacity() + words_.capacity()) * sizeof(int) + // lcp_.capacity() * sizeof(int16_t) + // - words_len_.capacity() * sizeof(word_len_type) + // + words_len_.capacity() * sizeof(WordsLenContainer::value_type) + // mapped_.capacity() * sizeof(V) + text_.capacity(); } @@ -224,7 +225,7 @@ class [[nodiscard]] suffix_map { std::vector sa_, words_; std::vector lcp_; - std::vector words_len_; + WordsLenContainer words_len_; std::vector mapped_; std::vector text_; bool built_ = false; diff --git a/cpp_src/gtests/tests/unit/ft/ft_generic.cc b/cpp_src/gtests/tests/unit/ft/ft_generic.cc index 9b3c7831a..a99d3a812 100644 --- a/cpp_src/gtests/tests/unit/ft/ft_generic.cc +++ b/cpp_src/gtests/tests/unit/ft/ft_generic.cc @@ -1360,6 +1360,75 @@ TEST_P(FTGenericApi, PartialMatchRank) { CheckAllPermutations("@", {"ft1^1.1", "ft2^1"}, " ТНТ*", {{"", "!ТНТ!"}, {"!ТНТ4!", ""}}, true, ", "); } +TEST_P(FTGenericApi, PartialMatchRankUsesUtf8CharLength) { + auto ftCfg = GetDefaultConfig(); + ftCfg.stopWords.clear(); + ftCfg.partialMatchDecrease = 90; + Init(ftCfg); + + const int exactId = Add("на"sv).second; + const int prefixId = Add("нат"sv).second; + const int suffixId = Add("она"sv).second; + const int containsId = Add("онат"sv).second; + + auto rankFor = [this](std::string_view dsl, int id) -> float { + auto query = reindexer::Query("nm1").Where("ft3", CondEq, std::string(dsl)).And().Where("id", CondEq, id).WithRank(); + auto qr = rt.Select(query); + EXPECT_EQ(qr.Count(), 1) << dsl << "; id=" << id; + if (qr.Count() != 1) { + return 0.0f; + } + return qr.begin().GetItemRefRanked().Rank().Value(); + }; + + const float exactRank = rankFor("на"sv, exactId); + ASSERT_GT(exactRank, 0.0f); + + const float prefixRank = rankFor("на*"sv, prefixId); + const float suffixRank = rankFor("*на"sv, suffixId); + const float containsRank = rankFor("*на*"sv, containsId); + + EXPECT_GT(prefixRank, exactRank * 0.6f); + EXPECT_GT(suffixRank, exactRank * 0.45f); + EXPECT_GT(suffixRank, ftCfg.rankingConfig.SuffixMin()); + EXPECT_GT(containsRank, exactRank * 0.25f); + EXPECT_GT(containsRank, ftCfg.rankingConfig.SuffixMin()); +} + +TEST_P(FTGenericApi, PartialMatchRankMinDenominatorUsesUtf8Chars) { + auto ftCfg = GetDefaultConfig(); + ftCfg.stopWords.clear(); + ftCfg.partialMatchDecrease = 90; + Init(ftCfg); + + const int latinExactId = Add("na"sv).second; + const int latinPrefixId = Add("nat"sv).second; + const int utf8ExactId = Add("на"sv).second; + const int utf8PrefixId = Add("нат"sv).second; + + auto rankFor = [this](std::string_view dsl, int id) -> float { + auto query = reindexer::Query("nm1").Where("ft3", CondEq, std::string(dsl)).And().Where("id", CondEq, id).WithRank(); + auto qr = rt.Select(query); + EXPECT_EQ(qr.Count(), 1) << dsl << "; id=" << id; + if (qr.Count() != 1) { + return 0.0f; + } + return qr.begin().GetItemRefRanked().Rank().Value(); + }; + + const float latinExactRank = rankFor("na"sv, latinExactId); + const float latinPrefixRank = rankFor("na*"sv, latinPrefixId); + const float utf8ExactRank = rankFor("на"sv, utf8ExactId); + const float utf8PrefixRank = rankFor("на*"sv, utf8PrefixId); + ASSERT_GT(latinExactRank, 0.0f); + ASSERT_GT(utf8ExactRank, 0.0f); + + const float latinPrefixRatio = latinPrefixRank / latinExactRank; + const float utf8PrefixRatio = utf8PrefixRank / utf8ExactRank; + EXPECT_NEAR(utf8PrefixRatio, latinPrefixRatio, 0.05f); + EXPECT_GT(utf8PrefixRatio, 0.6f); +} + TEST_P(FTGenericApi, PrefixLongUtf8Word) { auto ftCfg = GetDefaultConfig(); ftCfg.stopWords.clear();