Repository navigation
Expand file tree
/
Copy pathsearch_query.py
More file actions
398 lines (318 loc) · 14.4 KB
/
Copy pathsearch_query.py
File metadata and controls
398 lines (318 loc) · 14.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
"""Helpers for search query handling across FTS and LIKE fallback paths."""
from __future__ import annotations
import re
import unicodedata
from typing import Callable, List
_CJK_RE = re.compile(
r"["
r"\u3400-\u4dbf"
r"\u4e00-\u9fff"
r"\u3000-\u303f"
r"\u3040-\u30ff"
r"\uac00-\ud7af"
r"\uff00-\uffef"
r"]"
)
_EMOJI_RE = re.compile(
r"["
r"\u2600-\u27bf"
r"\U0001F300-\U0001FAFF"
r"]"
)
_QUOTED_PHRASE_RE = re.compile(r'"([^"]+)"')
_BOOLEAN_OPERATORS = {"AND", "OR", "NOT", "NEAR"}
_RISKY_FTS_TOKEN_RE = re.compile(r"[A-Za-z0-9][\-:/][A-Za-z0-9]")
_SPLIT_PUNCT_RE = re.compile(r"[-:/]+")
_STRIP_EDGE_PUNCT = "\"'()[]{}.,;"
# Characters that are special in FTS5 QUERY SYNTAX (as opposed to characters
# FTS5 simply cannot spell in a bareword). Only these have to go on the LIKE
# path, which has no query grammar of its own.
_FTS5_SPECIAL_CHARS = frozenset('"()*^-:{}.')
def _like_safe_char(char: str) -> str:
"""Map one unquoted character to its LIKE-safe form.
LIKE is a substring match, so the only thing it needs removed is the
operator punctuation a user typed FOR the index (quoted phrases, prefix
``*``). Everything else is signal it can match on — emoji above all, which
the FTS term form must drop because unicode61 does not index it, and which
LIKE is therefore the ONLY way to find.
"""
return " " if char in _FTS5_SPECIAL_CHARS else char
def _fts5_safe_char(char: str) -> str:
"""Map one unquoted character to its FTS5 bareword-safe form.
An FTS5 bareword accepts only alphanumerics; every other character is
either query syntax (``"()*^-:{}.``), a string delimiter (``'``), or a
plain syntax error (``? , & $ ! % = < ;`` ...). A raw natural-language
question therefore fails ``MATCH`` outright and used to fall through to the
LIKE full-scan, which blows the recall deadline at scale and returns
nothing (F31 §3). Substituting a separator instead lets a question reach
the index in its term form; the default unicode61 tokenizer splits the
INDEXED text on exactly the same boundary, so no term is lost by the
substitution.
``str.isalnum()`` alone is NOT that boundary: a combining mark is not
alphanumeric, so a decomposed ``naïve`` (``nai`` + U+0308) split into
``nai ve`` while unicode61 indexes the word as ``naive`` — zero rows where
the raw query matched. Marks therefore stay inside the token, and
``sanitize_fts5_query`` composes the query first.
"""
if char.isalnum() or char.isspace():
return char
return char if unicodedata.category(char).startswith("M") else " "
def _lower_operator_tokens(text: str) -> str:
return " ".join(
token.lower() if token in _BOOLEAN_OPERATORS else token
for token in text.split(" ")
)
def _neutralize_bare_operators(sanitized: str) -> str:
"""Lowercase bare AND/OR/NOT/NEAR outside quoted phrases.
FTS5 operators are only operators in UPPERCASE, so lowercasing turns them
back into ordinary barewords. Without this a raw question SILENTLY ACQUIRES
boolean semantics it never asked for: ``Portland, OR hotel`` sanitized to
``Portland OR hotel`` and broadened to a disjunction, while a leading
``NOT ready`` was a syntax error that dumped the query onto the LIKE
full-scan. Raw and deliberate queries share one unmarked entry point, so the
raw reading has to be the safe one. Quoted phrases are left alone: an
explicit ``"NEAR"`` is already a literal, not an operator.
"""
out: list[str] = []
last = 0
for match in _QUOTED_PHRASE_RE.finditer(sanitized):
out.append(_lower_operator_tokens(sanitized[last:match.start()]))
out.append(match.group(0))
last = match.end()
out.append(_lower_operator_tokens(sanitized[last:]))
return "".join(out)
def sanitize_fts5_query(query: str, *, allow_operators: bool = False) -> str:
"""Reduce a query to FTS5-safe terms, preserving balanced phrase quotes.
Composed (NFC) first so a decomposed accent is one alphanumeric character
rather than a base plus a combining mark, which is what unicode61 folds and
indexes. The LIKE path deliberately does NOT normalize: it is a literal
substring match against stored bytes, so it must not re-spell the query.
``allow_operators`` is the explicit marker for a query a CALLER composed as
FTS5 syntax (the benchmark harness joins its barewords with ``OR``). It
keeps bare AND/OR/NOT/NEAR intact. It must never be set for text that came
from a user or an agent: the default assumes raw prose, which is the only
safe reading when the two cannot be told apart.
"""
composed = unicodedata.normalize("NFC", query or "")
sanitized = _sanitize_query(composed, _fts5_safe_char)
return sanitized if allow_operators else _neutralize_bare_operators(sanitized)
def sanitize_like_query(query: str) -> str:
"""Strip FTS5 syntax operators, preserving every other character.
The LIKE path's sanitization has to be WEAKER than the FTS one: a character
the index cannot spell is still a character LIKE can match. Sharing the FTS
term form here dropped emoji from the fallback that exists to find them
(``launch 🚀`` searched only ``%launch%``).
"""
return _sanitize_query(query, _like_safe_char)
def _sanitize_query(query: str, replace: Callable[[str], str]) -> str:
"""Walk ``query`` outside balanced phrase quotes, mapping chars via ``replace``."""
if not query:
return ""
result: list[str] = []
quote_buffer: list[str] = []
in_quote = False
for char in query:
if char == '"':
if in_quote:
result.append('"')
result.extend(quote_buffer)
result.append('"')
quote_buffer = []
in_quote = False
else:
if result and not result[-1].isspace():
result.append(" ")
in_quote = True
quote_buffer = []
continue
if in_quote:
quote_buffer.append(char)
continue
result.append(replace(char))
if in_quote and quote_buffer:
result.extend(replace(char) for char in "".join(quote_buffer))
return " ".join("".join(result).split())
_WORD_RE = re.compile(r"[\w-]+", re.UNICODE)
def contains_cjk(text: str) -> bool:
return bool(_CJK_RE.search(text or ""))
def contains_emoji(text: str) -> bool:
return bool(_EMOJI_RE.search(text or ""))
def contains_risky_fts_ascii(text: str) -> bool:
raw = (text or "").strip()
if not raw:
return False
if raw.count('"') % 2:
return True
text_without_phrases = _QUOTED_PHRASE_RE.sub(" ", raw)
return bool(_RISKY_FTS_TOKEN_RE.search(text_without_phrases))
def requires_like_fallback(query: str, sanitized: str | None = None) -> bool:
"""Whether ``query`` must be answered by the LIKE scan instead of the index.
The test is what SANITIZATION LOSES, not what the FTS5 query grammar cannot
spell. A compound token (``art-related``, ``api:v2``, ``a/b``) sanitizes to
ordinary terms the index answers perfectly well, so routing it to the
full-table LIKE scan just re-imports the scaling ceiling this branch is
fixing — 6 of the 50 fixed Phase 1B questions carry a hyphen. The risky-ASCII
check therefore runs against the SANITIZED form, which is what actually
reaches ``MATCH``.
Genuine losses stay on LIKE: unicode61 does not segment CJK, it drops emoji
from the index entirely, and a query that sanitizes to nothing has no terms
left to match. Those two character classes are tested against the RAW query
because sanitization is exactly what removes them.
"""
raw = query or ""
safe = sanitize_fts5_query(raw) if sanitized is None else (sanitized or "")
if not safe.strip():
return True
if contains_cjk(raw) or contains_emoji(raw):
return True
return contains_risky_fts_ascii(safe)
def _token_variants(token: str) -> List[str]:
cleaned = (token or "").strip().strip(_STRIP_EDGE_PUNCT)
if not cleaned:
return []
if cleaned.upper() in _BOOLEAN_OPERATORS:
return []
variants = [cleaned]
if _SPLIT_PUNCT_RE.search(cleaned):
parts = [part for part in _SPLIT_PUNCT_RE.split(cleaned) if part]
if len(parts) > 1:
variants.extend(parts)
deduped: list[str] = []
seen: set[str] = set()
for variant in variants:
if variant not in seen:
deduped.append(variant)
seen.add(variant)
return deduped
def extract_search_terms(query: str) -> List[str]:
text = (query or "").strip()
if not text:
return []
terms: list[str] = []
for phrase in _QUOTED_PHRASE_RE.findall(text):
cleaned = phrase.strip()
if cleaned:
terms.append(cleaned)
text_without_phrases = _QUOTED_PHRASE_RE.sub(" ", text)
for token in text_without_phrases.split():
terms.extend(_token_variants(token))
if not terms:
fallback_text = text.strip().strip(_STRIP_EDGE_PUNCT)
if fallback_text:
terms.append(fallback_text)
deduped: list[str] = []
seen: set[str] = set()
for term in terms:
if term not in seen:
deduped.append(term)
seen.add(term)
return deduped
def extract_quoted_phrases(query: str) -> List[str]:
return [phrase.strip() for phrase in _QUOTED_PHRASE_RE.findall(query or "") if phrase.strip()]
def escape_like(term: str) -> str:
return term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
def count_term_matches(text: str, term: str) -> int:
haystack = (text or "")
needle = (term or "")
if not haystack or not needle:
return 0
return haystack.lower().count(needle.lower())
def compute_directness_score(text: str, terms: List[str], phrases: List[str] | None = None) -> float:
content = text or ""
if not content:
return 0.0
unique_hits = 0
total_hits = 0
non_phrase_unique_hits = 0
non_phrase_total_hits = 0
normalized_phrases = {(phrase or "").strip().lower() for phrase in (phrases or []) if (phrase or "").strip()}
for term in terms:
matches = count_term_matches(content, term)
if matches > 0:
unique_hits += 1
total_hits += matches
if term.strip().lower() not in normalized_phrases:
non_phrase_unique_hits += 1
non_phrase_total_hits += matches
phrase_hits = 0
lowered = content.lower()
for phrase in phrases or []:
if phrase and phrase.lower() in lowered:
phrase_hits += 1
repetition_penalty = max(0, total_hits - unique_hits)
non_phrase_repetition_penalty = max(0, non_phrase_total_hits - non_phrase_unique_hits)
score = float((unique_hits * 5) + (phrase_hits * 8))
if not phrases:
score -= min(repetition_penalty, 6)
else:
score -= min(non_phrase_repetition_penalty, 6)
if phrases:
for phrase in phrases:
normalized_phrase = (phrase or "").strip().lower()
if not normalized_phrase:
continue
phrase_occurrences = lowered.count(normalized_phrase)
if phrase_occurrences <= 1:
continue
segments = re.split(re.escape(normalized_phrase), lowered)
gap_unique_counts = []
for segment in segments:
segment_tokens = [
token.lower()
for token in _WORD_RE.findall(segment)
if any(char.isalpha() for char in token)
]
gap_unique_counts.append(len(set(segment_tokens)))
interior_gap_counts = gap_unique_counts[1:-1]
tail_gap_count = gap_unique_counts[-1] if gap_unique_counts else 0
extra_occurrences = phrase_occurrences - 1
score -= extra_occurrences * 0.5
score -= sum(1.5 for count in interior_gap_counts if 0 < count <= 4)
if all(count == 0 for count in interior_gap_counts) and tail_gap_count <= 2:
score -= min(extra_occurrences, 3) * 1.0
return score
def _is_precise_query_shape(terms: List[str], phrases: List[str] | None = None) -> bool:
if len(terms) == 1:
return True
return len(phrases or []) == 1 and len(terms) <= 2
def should_widen_candidate_fetch(terms: List[str], phrases: List[str] | None = None) -> bool:
return _is_precise_query_shape(terms, phrases)
def should_apply_directness_rank_adjustment(terms: List[str], phrases: List[str] | None = None) -> bool:
return _is_precise_query_shape(terms, phrases)
def compute_directness_rank_bonus_upper_bound(terms: List[str], phrases: List[str] | None = None) -> float:
return float((len(terms) * 5) + (len(phrases or []) * 8))
def compute_search_fetch_limit(limit: int, terms: List[str], phrases: List[str] | None = None) -> int:
base = max(limit * 5, limit, 20)
if should_widen_candidate_fetch(terms, phrases):
return max(base, limit * 10, 50)
return base
def compute_like_fallback_fetch_limit(limit: int, terms: List[str], phrases: List[str] | None = None) -> int:
"""Bound LIKE fallback candidate rows before Python-side scoring/sorting."""
return compute_search_fetch_limit(limit, terms, phrases)
def compute_search_candidate_cap(limit: int) -> int:
"""Return a hard upper bound on candidate rows inspected per search call."""
return min(max(limit * 20, limit, 500), 5_000)
AGE_DECAY_RATE = 0.001
def normalize_search_sort(sort: str | None) -> str:
"""Normalize sort parameter to one of: recency, relevance, hybrid."""
normalized = (sort or "recency").strip().lower()
return normalized if normalized in {"recency", "relevance", "hybrid"} else "recency"
def build_snippet(text: str, terms: List[str], width: int = 80) -> str:
content = (text or "")
if not content:
return ""
lowered = content.lower()
for term in terms:
if not term:
continue
idx = lowered.find(term.lower())
if idx >= 0:
start = max(0, idx - width // 2)
end = min(len(content), idx + len(term) + width // 2)
snippet = content[start:end]
if start > 0:
snippet = "..." + snippet
if end < len(content):
snippet = snippet + "..."
return snippet
return content[:width] + ("..." if len(content) > width else "")