Fix UTF-8 partial-match ranking

cpp_src/core/ft/ft_fast/selecterimpl.h

@@ -293,6 +293,7 @@ ft::TermResults<IdCont> Selector<IdCont>::buildTermResults(const FtDSLEntry& ter
 		size_t matched = 0, vids = 0, excludedCnt = 0;
 		const std::string& patternUtf8 = variant.PatternUtf8();
 		const size_t patternBytes = patternUtf8.length();
+		const auto patternChars = static_cast<uint32_t>(getUTF8StringCharactersCount(patternUtf8));
 		for (const auto& step : holder_.steps) {
 			if (variant.lowRelevance && totalVids >= lowRelevanceLimit) {
 				break;
@@ -329,11 +330,12 @@ ft::TermResults<IdCont> Selector<IdCont>::buildTermResults(const FtDSLEntry& ter
 					break;
 				}
 
-				// ToDo fix it (broken for russian utf8 symbols)
-				const int matchDif = std::abs(long(word.length() - variant.PatternUtf8().length() + wordLengthBeforePattern));
+				const auto wordChars = static_cast<uint32_t>(getUTF8StringCharactersCount(word));
+				const uint32_t unmatchedChars = wordChars > patternChars ? wordChars - patternChars : 0;
 				const float boost = std::max(getTermBoost(std::string(word)), variant.boost);
-				const float decreasePenalty = static_cast<float>(holder_.cfg_->partialMatchDecrease * matchDif) /
-											  std::max<float>(variant.PatternUtf8().length(), kMinPartialMatchDenominator);
+				const float decreasePenalty =
+					(static_cast<float>(holder_.cfg_->partialMatchDecrease) * static_cast<float>(unmatchedChars)) /
+					static_cast<float>(std::max(patternChars, kMinPartialMatchDenominator));
 				float proc = std::max<float>(variant.proc - decreasePenalty, isPrefix ? rankingCfg.PrefixMin() : rankingCfg.SuffixMin());
 				proc = std::min<float>(proc, variant.proc);
 				if (boost > 0.0f) {

cpp_src/estl/suffix_map.h

@@ -18,6 +18,7 @@ class [[nodiscard]] suffix_map {
 private:
 	typedef size_t size_type;
 	typedef unsigned char char_type;
+	using WordsLenContainer = std::vector<word_len_type>;
 
 	class [[nodiscard]] value_type : public std::pair<const CharT*, V> {
 	public:
@@ -194,7 +195,7 @@ class [[nodiscard]] suffix_map {
 	size_t heap_size() noexcept {
 		return (sa_.capacity() + words_.capacity()) * sizeof(int) +	 //
 			   lcp_.capacity() * sizeof(int16_t) +					 //
-			   words_len_.capacity() * sizeof(word_len_type) +		 //
+			   words_len_.capacity() * sizeof(WordsLenContainer::value_type) +	 //
 			   mapped_.capacity() * sizeof(V) + text_.capacity();
 	}
 
@@ -224,7 +225,7 @@ class [[nodiscard]] suffix_map {
 
 	std::vector<int> sa_, words_;
 	std::vector<int16_t> lcp_;
-	std::vector<word_len_type> words_len_;
+	WordsLenContainer words_len_;
 	std::vector<V> mapped_;
 	std::vector<CharT> text_;
 	bool built_ = false;

cpp_src/gtests/tests/unit/ft/ft_generic.cc

@@ -1360,6 +1360,75 @@ TEST_P(FTGenericApi, PartialMatchRank) {
 	CheckAllPermutations("@", {"ft1^1.1", "ft2^1"}, " ТНТ*", {{"", "!ТНТ!"}, {"!ТНТ4!", ""}}, true, ", ");
 }
 
+TEST_P(FTGenericApi, PartialMatchRankUsesUtf8CharLength) {
+	auto ftCfg = GetDefaultConfig();
+	ftCfg.stopWords.clear();
+	ftCfg.partialMatchDecrease = 90;
+	Init(ftCfg);
+
+	const int exactId = Add("на"sv).second;
+	const int prefixId = Add("нат"sv).second;
+	const int suffixId = Add("она"sv).second;
+	const int containsId = Add("онат"sv).second;
+
+	auto rankFor = [this](std::string_view dsl, int id) -> float {
+		auto query = reindexer::Query("nm1").Where("ft3", CondEq, std::string(dsl)).And().Where("id", CondEq, id).WithRank();
+		auto qr = rt.Select(query);
+		EXPECT_EQ(qr.Count(), 1) << dsl << "; id=" << id;
+		if (qr.Count() != 1) {
+			return 0.0f;
+		}
+		return qr.begin().GetItemRefRanked().Rank().Value();
+	};
+
+	const float exactRank = rankFor("на"sv, exactId);
+	ASSERT_GT(exactRank, 0.0f);
+
+	const float prefixRank = rankFor("на*"sv, prefixId);
+	const float suffixRank = rankFor("*на"sv, suffixId);
+	const float containsRank = rankFor("*на*"sv, containsId);
+
+	EXPECT_GT(prefixRank, exactRank * 0.6f);
+	EXPECT_GT(suffixRank, exactRank * 0.45f);
+	EXPECT_GT(suffixRank, ftCfg.rankingConfig.SuffixMin());
+	EXPECT_GT(containsRank, exactRank * 0.25f);
+	EXPECT_GT(containsRank, ftCfg.rankingConfig.SuffixMin());
+}
+
+TEST_P(FTGenericApi, PartialMatchRankMinDenominatorUsesUtf8Chars) {
+	auto ftCfg = GetDefaultConfig();
+	ftCfg.stopWords.clear();
+	ftCfg.partialMatchDecrease = 90;
+	Init(ftCfg);
+
+	const int latinExactId = Add("na"sv).second;
+	const int latinPrefixId = Add("nat"sv).second;
+	const int utf8ExactId = Add("на"sv).second;
+	const int utf8PrefixId = Add("нат"sv).second;
+
+	auto rankFor = [this](std::string_view dsl, int id) -> float {
+		auto query = reindexer::Query("nm1").Where("ft3", CondEq, std::string(dsl)).And().Where("id", CondEq, id).WithRank();
+		auto qr = rt.Select(query);
+		EXPECT_EQ(qr.Count(), 1) << dsl << "; id=" << id;
+		if (qr.Count() != 1) {
+			return 0.0f;
+		}
+		return qr.begin().GetItemRefRanked().Rank().Value();
+	};
+
+	const float latinExactRank = rankFor("na"sv, latinExactId);
+	const float latinPrefixRank = rankFor("na*"sv, latinPrefixId);
+	const float utf8ExactRank = rankFor("на"sv, utf8ExactId);
+	const float utf8PrefixRank = rankFor("на*"sv, utf8PrefixId);
+	ASSERT_GT(latinExactRank, 0.0f);
+	ASSERT_GT(utf8ExactRank, 0.0f);
+
+	const float latinPrefixRatio = latinPrefixRank / latinExactRank;
+	const float utf8PrefixRatio = utf8PrefixRank / utf8ExactRank;
+	EXPECT_NEAR(utf8PrefixRatio, latinPrefixRatio, 0.05f);
+	EXPECT_GT(utf8PrefixRatio, 0.6f);
+}
+
 TEST_P(FTGenericApi, PrefixLongUtf8Word) {
 	auto ftCfg = GetDefaultConfig();
 	ftCfg.stopWords.clear();