From 0a3a515fc26e75377eedff4cc0331d507de6d083 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:48:09 -0400 Subject: [PATCH 01/48] contract(parser): carry native parse output --- .../perl-parser/src/incremental/diagnostics.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/perl-parser/src/incremental/diagnostics.rs b/crates/perl-parser/src/incremental/diagnostics.rs index 5729baf62c..b434b9642a 100644 --- a/crates/perl-parser/src/incremental/diagnostics.rs +++ b/crates/perl-parser/src/incremental/diagnostics.rs @@ -1,13 +1,29 @@ use lsp_types::Diagnostic; +use perl_parser_core::error::ParseOutput; use std::ops::Range; -/// Result of incremental reparse +/// Result of incremental reparse. #[derive(Debug)] #[non_exhaustive] pub struct ReparseResult { + /// Byte ranges reparsed or replaced by the selected strategy. pub changed_ranges: Vec>, + /// Authoritative native parser output for the current source generation. + /// + /// This carries the AST, ordered parser diagnostics, recovery count, + /// budget usage, and early-termination state produced by the same + /// `Parser::parse_with_recovery` contract used by a fresh parse. + pub parse_output: ParseOutput, + /// Legacy LSP-shaped diagnostics retained for compatibility. + /// + /// Parser consumers should use [`Self::parse_output`]. LSP projection is a + /// transport concern and remains intentionally separate from the native + /// parser output contract. pub diagnostics: Vec, + /// Number of source bytes covered by reparsing work. pub reparsed_bytes: usize, + /// Number of lexer tokens retained from the previous state. pub reused_tokens: usize, + /// Total token count in the resulting incremental state. pub token_count: usize, } From f428f28788357a52bf974ed5c0ab0845fd7567b9 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:48:33 -0400 Subject: [PATCH 02/48] fix(parser): preserve recovered incremental output --- crates/perl-parser/src/incremental/state.rs | 68 +++++++++++++-------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/crates/perl-parser/src/incremental/state.rs b/crates/perl-parser/src/incremental/state.rs index acabda51e9..db92eb8b47 100644 --- a/crates/perl-parser/src/incremental/state.rs +++ b/crates/perl-parser/src/incremental/state.rs @@ -2,7 +2,8 @@ use crate::incremental::checkpoint::{LexCheckpoint, ParseCheckpoint, ScopeSnapsh use crate::incremental::lex::create_lex_checkpoints; use perl_lexer::{PerlLexer, Token, TokenType}; use perl_line_index::LineIndex; -use perl_parser_core::ast::{Node, NodeKind, SourceLocation}; +use perl_parser_core::ast::{Node, NodeKind}; +use perl_parser_core::error::ParseOutput; use perl_parser_core::parser::Parser; use ropey::Rope; @@ -12,39 +13,32 @@ pub struct IncrementalState { pub line_index: LineIndex, pub lex_checkpoints: Vec, pub parse_checkpoints: Vec, - /// Parsed AST. + /// Authoritative native parser output for the current source. /// - /// **Staleness invariant (#5036):** This field is only guaranteed current - /// after `new()` or `full_reparse()`. After `apply_single_edit()` it may be - /// stale relative to `tokens`/`source`. It is kept solely for - /// `create_parse_checkpoints` in `full_reparse`. Production providers - /// always use a fresh full reparse via `ParsedSnapshot`, never this field. - #[deprecated( - note = "Only valid after new()/full_reparse(); may be stale after apply_single_edit(). Use a fresh parse instead." - )] + /// This is produced by `Parser::parse_with_recovery` and carries the AST, + /// ordered parser diagnostics, recovery count, budget usage, and early + /// termination state. Incremental consumers should use this field rather + /// than reconstructing parser state from an AST alone. + pub parse_output: ParseOutput, + /// Parsed AST compatibility field. + /// + /// This field mirrors [`Self::parse_output`]'s AST after every supported + /// state transition. It remains temporarily for compatibility with existing + /// callers and parse-checkpoint code. + #[deprecated(note = "Use parse_output.ast; this compatibility mirror will be removed.")] pub ast: Node, pub tokens: Vec, pub source: String, } impl IncrementalState { - #[expect(deprecated, reason = "the legacy AST field seeds parse checkpoints for compatibility")] + #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] pub fn new(source: String) -> Self { let rope = Rope::from_str(&source); let line_index = LineIndex::new(&source); let mut parser = Parser::new(&source); - let ast = match parser.parse() { - Ok(ast) => ast, - Err(e) => Node::new( - NodeKind::Error { - message: e.to_string(), - expected: vec![], - found: None, - partial: None, - }, - SourceLocation { start: 0, end: source.len() }, - ), - }; + let parse_output = parser.parse_with_recovery(); + let ast = parse_output.ast.clone(); let mut lexer = PerlLexer::new(&source); let mut tokens = Vec::new(); while let Some(token) = lexer.next_token() { @@ -54,16 +48,40 @@ impl IncrementalState { tokens.push(token); } let lex_checkpoints = create_lex_checkpoints(&tokens, &line_index); - let parse_checkpoints = Self::create_parse_checkpoints(&ast); - Self { rope, line_index, lex_checkpoints, parse_checkpoints, ast, tokens, source } + let parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast); + Self { + rope, + line_index, + lex_checkpoints, + parse_checkpoints, + parse_output, + ast, + tokens, + source, + } } + pub fn find_lex_checkpoint(&self, byte: usize) -> Option<&LexCheckpoint> { self.lex_checkpoints.iter().rev().find(|cp| cp.byte <= byte) } + pub fn find_parse_checkpoint(&self, byte: usize) -> Option<&ParseCheckpoint> { self.parse_checkpoints.iter().rev().find(|cp| cp.byte <= byte) } + /// Refresh the authoritative parser output from the current source. + /// + /// The compatibility AST and parse checkpoints are updated from the same + /// recovered parse so the state cannot expose mixed parse generations. + #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] + pub(crate) fn refresh_parse_output(&mut self) { + let mut parser = Parser::new(&self.source); + let parse_output = parser.parse_with_recovery(); + self.parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast); + self.ast = parse_output.ast.clone(); + self.parse_output = parse_output; + } + pub(crate) fn create_parse_checkpoints(ast: &Node) -> Vec { let mut checkpoints = vec![]; let mut scope = ScopeSnapshot::default(); From f9f8bc98ac7b2d320675f7ab9b0a741c0e825065 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:49:07 -0400 Subject: [PATCH 03/48] fix(parser): refresh native output on fallback --- crates/perl-parser/src/incremental/reparse.rs | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/crates/perl-parser/src/incremental/reparse.rs b/crates/perl-parser/src/incremental/reparse.rs index 1314a58bbc..6ec03b595a 100644 --- a/crates/perl-parser/src/incremental/reparse.rs +++ b/crates/perl-parser/src/incremental/reparse.rs @@ -3,8 +3,6 @@ use crate::incremental::{ }; use anyhow::Result; use perl_lexer::{PerlLexer, TokenType}; -use perl_parser_core::ast::{Node, NodeKind, SourceLocation}; -use perl_parser_core::parser::Parser; use ropey::Rope; use std::ops::Range; @@ -119,21 +117,8 @@ pub(crate) fn apply_single_edit( Ok(SingleEditReparse { range: cp.byte..last, reused_tokens, token_count: state.tokens.len() }) } -#[expect(deprecated, reason = "full reparse is the legacy AST field's supported refresh boundary")] pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result { - let mut parser = Parser::new(&state.source); - state.ast = match parser.parse() { - Ok(ast) => ast, - Err(e) => Node::new( - NodeKind::Error { - message: e.to_string(), - expected: vec![], - found: None, - partial: None, - }, - SourceLocation { start: 0, end: state.source.len() }, - ), - }; + state.refresh_parse_output(); let mut lexer = PerlLexer::new(&state.source); let mut tokens = Vec::new(); while let Some(token) = lexer.next_token() { @@ -146,9 +131,9 @@ pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result Date: Tue, 11 Aug 2026 05:49:27 -0400 Subject: [PATCH 04/48] fix(parser): return current parse output after edits --- crates/perl-parser/src/incremental/mod.rs | 41 ++++------------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/crates/perl-parser/src/incremental/mod.rs b/crates/perl-parser/src/incremental/mod.rs index 059bc428ac..ec5e466163 100644 --- a/crates/perl-parser/src/incremental/mod.rs +++ b/crates/perl-parser/src/incremental/mod.rs @@ -11,8 +11,6 @@ mod state; mod strategy; use anyhow::Result; -use perl_parser_core::ast::{Node, NodeKind, SourceLocation}; -use perl_parser_core::parser::Parser; pub use perl_line_index::LineIndex; @@ -34,7 +32,7 @@ pub mod incremental_integration; pub mod incremental_simple; pub mod incremental_v2; -/// Apply edits incrementally +/// Apply edits incrementally. pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result { let mut sorted_edits = edits.to_vec(); sorted_edits.sort_by_key(|e| e.start_byte); @@ -60,14 +58,14 @@ pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result Result ast, - Err(e) => Node::new( - NodeKind::Error { - message: e.to_string(), - expected: vec![], - found: None, - partial: None, - }, - SourceLocation { start: 0, end: state.source.len() }, - ), - }; - state.parse_checkpoints = IncrementalState::create_parse_checkpoints(&state.ast); -} - #[cfg(test)] mod tests; From 735c802330b24f1944e2da6624c28711fe81fb7f Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:50:16 -0400 Subject: [PATCH 05/48] test(parser): prove incremental recovery output parity --- .../tests/incremental_parse_output.rs | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 crates/perl-parser/tests/incremental_parse_output.rs diff --git a/crates/perl-parser/tests/incremental_parse_output.rs b/crates/perl-parser/tests/incremental_parse_output.rs new file mode 100644 index 0000000000..478a7ad6e0 --- /dev/null +++ b/crates/perl-parser/tests/incremental_parse_output.rs @@ -0,0 +1,103 @@ +#![cfg(feature = "incremental")] +//! Differential tests for the incremental native parse-output contract. + +use perl_parser::{Edit, IncrementalState, ParseOutput, Parser, apply_edits}; + +type TestResult = Result<(), Box>; + +fn fresh_output(source: &str) -> ParseOutput { + let mut parser = Parser::new(source); + parser.parse_with_recovery() +} + +fn assert_output_equivalent(actual: &ParseOutput, expected: &ParseOutput) { + assert_eq!(actual.ast, expected.ast, "AST differs from a fresh recovered parse"); + assert_eq!( + actual.diagnostics, expected.diagnostics, + "ordered parser diagnostics differ from a fresh recovered parse" + ); + assert_eq!(actual.terminated_early, expected.terminated_early); + assert_eq!(actual.recovered_count, expected.recovered_count); + assert_eq!(actual.budget_usage.errors_emitted, expected.budget_usage.errors_emitted); + assert_eq!(actual.budget_usage.current_depth, expected.budget_usage.current_depth); + assert_eq!(actual.budget_usage.max_depth_reached, expected.budget_usage.max_depth_reached); + assert_eq!(actual.budget_usage.tokens_skipped, expected.budget_usage.tokens_skipped); + assert_eq!( + actual.budget_usage.recoveries_attempted, + expected.budget_usage.recoveries_attempted + ); +} + +fn apply_reference_edit(source: &str, edit: &Edit) -> Result { + if edit.start_byte > edit.old_end_byte || edit.old_end_byte > source.len() { + return Err("reference edit range is out of bounds"); + } + if !source.is_char_boundary(edit.start_byte) || !source.is_char_boundary(edit.old_end_byte) { + return Err("reference edit range is not on UTF-8 boundaries"); + } + + let mut result = source.to_string(); + result.replace_range(edit.start_byte..edit.old_end_byte, &edit.new_text); + Ok(result) +} + +#[test] +fn initial_malformed_state_keeps_the_native_recovered_tree_and_diagnostics() -> TestResult { + let source = "my $x = ; print 1;"; + let state = IncrementalState::new(source.to_string()); + let fresh = fresh_output(source); + + assert!(!fresh.diagnostics.is_empty(), "fixture must exercise structured recovery"); + assert_output_equivalent(&state.parse_output, &fresh); + + Ok(()) +} + +#[test] +fn clean_to_malformed_edit_returns_the_current_native_parse_output() -> TestResult { + let source = "my $x = 1; print 2;"; + let start = source.find("= 1").ok_or("clean fixture lost its initializer")? + 2; + let edit = Edit { + start_byte: start, + old_end_byte: start + 1, + new_end_byte: start, + new_text: String::new(), + }; + let final_source = apply_reference_edit(source, &edit)?; + let fresh = fresh_output(&final_source); + assert!(!fresh.diagnostics.is_empty(), "edited fixture must require recovery"); + + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(state.source, final_source); + assert_output_equivalent(&state.parse_output, &fresh); + assert_output_equivalent(&result.parse_output, &fresh); + + Ok(()) +} + +#[test] +fn malformed_to_clean_edit_removes_recovery_diagnostics_atomically() -> TestResult { + let source = "my $x = ; print 2;"; + let start = source.find("= ;").ok_or("malformed fixture lost its insertion point")? + 2; + let edit = Edit { + start_byte: start, + old_end_byte: start, + new_end_byte: start + 1, + new_text: "1".to_string(), + }; + let final_source = apply_reference_edit(source, &edit)?; + let fresh = fresh_output(&final_source); + assert!(fresh.diagnostics.is_empty(), "repaired fixture should parse cleanly"); + + let mut state = IncrementalState::new(source.to_string()); + assert!(!state.parse_output.diagnostics.is_empty()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(state.source, final_source); + assert_output_equivalent(&state.parse_output, &fresh); + assert_output_equivalent(&result.parse_output, &fresh); + + Ok(()) +} From 05d9a307d424853e149f19d8f24c19ed2fa7bd8e Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:50:41 -0400 Subject: [PATCH 06/48] ci(parser): run incremental parse-output parity --- scripts/ci/run_parser_integration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/ci/run_parser_integration.py b/scripts/ci/run_parser_integration.py index b351a46458..70ffdd7db6 100644 --- a/scripts/ci/run_parser_integration.py +++ b/scripts/ci/run_parser_integration.py @@ -101,7 +101,7 @@ def main() -> int: return result.returncode # Feature-gated parser tests are not exercised by the default target - # command. Keep this explicit proof in the same bounded parser gate. + # command. Keep these explicit proofs in the same bounded parser gate. incremental_command = [ "cargo", "test", @@ -112,6 +112,8 @@ def main() -> int: "incremental", "--test", "incremental_parser_accuracy", + "--test", + "incremental_parse_output", "--", "--test-threads=4", ] From 316504192b7da28ad7b20d0cf1257c781c553eb2 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:10:41 -0400 Subject: [PATCH 07/48] fix(parser): capture live lexer restart state --- crates/perl-parser/src/incremental/lex.rs | 211 ++++++++++++++++++---- 1 file changed, 176 insertions(+), 35 deletions(-) diff --git a/crates/perl-parser/src/incremental/lex.rs b/crates/perl-parser/src/incremental/lex.rs index 213bdf6c43..ef6c84aaad 100644 --- a/crates/perl-parser/src/incremental/lex.rs +++ b/crates/perl-parser/src/incremental/lex.rs @@ -1,42 +1,183 @@ use crate::incremental::LineIndex; use crate::incremental::checkpoint::LexCheckpoint; -use perl_lexer::{LexerMode, Token, TokenType}; +use anyhow::Result; +use perl_lexer::{ + Checkpointable, LexerCheckpoint as LiveLexerCheckpoint, PerlLexer, Token, TokenType, +}; -pub(crate) fn create_lex_checkpoints( - tokens: &[Token], +pub(crate) struct LexedSource { + pub(crate) tokens: Vec, + pub(crate) checkpoints: Vec, + #[cfg(test)] + pub(crate) live_checkpoints: Vec, +} + +fn summarize_checkpoint( + checkpoint: &LiveLexerCheckpoint, + line_index: &LineIndex, +) -> LexCheckpoint { + let (line, column) = line_index.byte_to_position(checkpoint.position); + LexCheckpoint { byte: checkpoint.position, mode: checkpoint.mode, line, column } +} + +/// Lex one complete source and capture restart candidates from the lexer's +/// actual live state before every emitted token and before terminal EOF. +pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) -> LexedSource { + let mut lexer = PerlLexer::new(source); + let mut tokens = Vec::new(); + let mut checkpoints = Vec::new(); + #[cfg(test)] + let mut live_checkpoints = Vec::new(); + + loop { + let live = lexer.checkpoint(); + checkpoints.push(summarize_checkpoint(&live, line_index)); + #[cfg(test)] + live_checkpoints.push(live); + + let Some(token) = lexer.next_token() else { + break; + }; + if token.token_type == TokenType::EOF { + break; + } + tokens.push(token); + } + + LexedSource { + tokens, + checkpoints, + #[cfg(test)] + live_checkpoints, + } +} + +/// Replay the old source to one previously captured token boundary and return +/// the complete live lexer checkpoint for that exact boundary. +/// +/// The public `LexCheckpoint` remains a compact compatibility summary. Restart +/// correctness is authorized only by this full `Checkpointable` state. +pub(crate) fn capture_live_checkpoint( + source: &str, + boundary: usize, +) -> Option { + let mut lexer = PerlLexer::new(source); + + loop { + let checkpoint = lexer.checkpoint(); + if checkpoint.position == boundary { + return Some(checkpoint); + } + if checkpoint.position > boundary { + return None; + } + + match lexer.next_token() { + Some(token) if token.token_type != TokenType::EOF => {} + _ => return None, + } + } +} + +/// Restore a complete live checkpoint into the edited source and re-lex from +/// that boundary to EOF. No old suffix is reused in this correctness-first +/// strategy. +pub(crate) fn lex_from_live_checkpoint( + source: &str, line_index: &LineIndex, -) -> Vec { - let mut checkpoints = - vec![LexCheckpoint { byte: 0, mode: LexerMode::ExpectTerm, line: 0, column: 0 }]; - let mut mode = LexerMode::ExpectTerm; - for token in tokens { - mode = match token.token_type { - TokenType::Semicolon | TokenType::LeftBrace | TokenType::RightBrace => { - let (line, column) = line_index.byte_to_position(token.end); - checkpoints.push(LexCheckpoint { - byte: token.end, - mode: LexerMode::ExpectTerm, - line, - column, - }); - LexerMode::ExpectTerm - } - TokenType::Keyword(ref kw) if kw.as_ref() == "sub" || kw.as_ref() == "package" => { - let (line, column) = line_index.byte_to_position(token.start); - checkpoints.push(LexCheckpoint { - byte: token.start, - mode: LexerMode::ExpectTerm, - line, - column, - }); - LexerMode::ExpectTerm - } - TokenType::Identifier(_) | TokenType::Number(_) | TokenType::StringLiteral => { - LexerMode::ExpectOperator - } - TokenType::Operator(_) => LexerMode::ExpectTerm, - _ => mode, + checkpoint: &LiveLexerCheckpoint, +) -> Result { + let mut lexer = PerlLexer::new(source); + if !lexer.can_restore(checkpoint) { + anyhow::bail!("live lexer checkpoint is not valid for the edited source"); + } + lexer.restore(checkpoint); + + let mut tokens = Vec::new(); + let mut checkpoints = Vec::new(); + #[cfg(test)] + let mut live_checkpoints = Vec::new(); + let mut last_position = checkpoint.position; + + loop { + let live = lexer.checkpoint(); + checkpoints.push(summarize_checkpoint(&live, line_index)); + #[cfg(test)] + live_checkpoints.push(live); + + let Some(token) = lexer.next_token() else { + break; + }; + if token.token_type == TokenType::EOF { + break; + } + if token.end <= last_position { + anyhow::bail!("incremental lexer did not advance at byte {}", token.start); } + last_position = token.end; + tokens.push(token); + } + + Ok(LexedSource { + tokens, + checkpoints, + #[cfg(test)] + live_checkpoints, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn live_checkpoint_preserves_after_arrow_state() -> Result<()> { + let source = "$object->method();"; + let line_index = LineIndex::new(source); + let lexed = lex_source_with_checkpoints(source, &line_index); + let method_index = lexed + .tokens + .iter() + .position(|token| token.text.as_ref() == "method") + .ok_or_else(|| anyhow::anyhow!("method token is missing"))?; + let checkpoint = &lexed.live_checkpoints[method_index]; + + assert_eq!(checkpoint.position, lexed.tokens[method_index].start); + assert!(checkpoint.after_arrow, "method restart must preserve after_arrow"); + Ok(()) + } + + #[test] + fn live_checkpoint_preserves_prototype_and_nesting_state() { + let source = "sub f($$) { return 1; }"; + let line_index = LineIndex::new(source); + let lexed = lex_source_with_checkpoints(source, &line_index); + + assert!( + lexed.live_checkpoints.iter().any(|checkpoint| checkpoint.in_prototype), + "prototype fixture must expose an in_prototype checkpoint" + ); + assert!( + lexed.live_checkpoints.iter().any(|checkpoint| checkpoint.paren_depth > 0), + "prototype fixture must expose parenthesis depth" + ); + } + + #[test] + fn replay_captures_the_same_complete_checkpoint() -> Result<()> { + let source = "$object->method();"; + let line_index = LineIndex::new(source); + let lexed = lex_source_with_checkpoints(source, &line_index); + let method_index = lexed + .tokens + .iter() + .position(|token| token.text.as_ref() == "method") + .ok_or_else(|| anyhow::anyhow!("method token is missing"))?; + let expected = &lexed.live_checkpoints[method_index]; + let replayed = capture_live_checkpoint(source, expected.position) + .ok_or_else(|| anyhow::anyhow!("live checkpoint replay failed"))?; + + assert_eq!(&replayed, expected); + Ok(()) } - checkpoints } From 99c67b045a84266893ae3be60b8d160b3ec4ae17 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:11:16 -0400 Subject: [PATCH 08/48] contract(parser): report lexer restart strategy --- .../src/incremental/diagnostics.rs | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/crates/perl-parser/src/incremental/diagnostics.rs b/crates/perl-parser/src/incremental/diagnostics.rs index b434b9642a..2fc356a410 100644 --- a/crates/perl-parser/src/incremental/diagnostics.rs +++ b/crates/perl-parser/src/incremental/diagnostics.rs @@ -2,6 +2,40 @@ use lsp_types::Diagnostic; use perl_parser_core::error::ParseOutput; use std::ops::Range; +/// Lexer work strategy selected for one incremental parse result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum LexRestartStrategy { + /// Lex the complete current source from byte zero. + FullRelex, + /// Restore one complete live lexer checkpoint and re-lex from there to EOF. + LiveCheckpointToEof, +} + +/// Truthful lexer restart and token-retention receipt. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct LexRestartReport { + /// Strategy that produced the current token stream. + pub strategy: LexRestartStrategy, + /// Byte boundary where fresh lexing began. + pub restart_byte: usize, + /// Number of source bytes lexed from the restart boundary to EOF. + pub relexed_bytes: usize, + /// Tokens before the restart boundary retained without re-lexing. + pub reused_prefix_tokens: usize, + /// Tokens after a synchronization boundary retained from the old suffix. + pub reused_suffix_tokens: usize, +} + +impl LexRestartReport { + /// Total old tokens retained by the selected strategy. + #[must_use] + pub fn reused_tokens(self) -> usize { + self.reused_prefix_tokens.saturating_add(self.reused_suffix_tokens) + } +} + /// Result of incremental reparse. #[derive(Debug)] #[non_exhaustive] @@ -20,9 +54,14 @@ pub struct ReparseResult { /// transport concern and remains intentionally separate from the native /// parser output contract. pub diagnostics: Vec, - /// Number of source bytes covered by reparsing work. + /// Lexer restart, fresh-work, and token-retention receipt. + pub lex_restart: LexRestartReport, + /// Number of source bytes covered by parser reparsing work. pub reparsed_bytes: usize, - /// Number of lexer tokens retained from the previous state. + /// Compatibility total of old lexer tokens retained from prefix and suffix. + /// + /// New consumers should use [`Self::lex_restart`] to distinguish prefix + /// retention from state-proven suffix reuse. pub reused_tokens: usize, /// Total token count in the resulting incremental state. pub token_count: usize, From cae41e17f5c6d3ad1fd5b06db6e423abe8d4afee Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:11:41 -0400 Subject: [PATCH 09/48] fix(parser): derive restart summaries from live lexing --- crates/perl-parser/src/incremental/state.rs | 23 +++++++++------------ 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/crates/perl-parser/src/incremental/state.rs b/crates/perl-parser/src/incremental/state.rs index db92eb8b47..711ed88f23 100644 --- a/crates/perl-parser/src/incremental/state.rs +++ b/crates/perl-parser/src/incremental/state.rs @@ -1,6 +1,6 @@ use crate::incremental::checkpoint::{LexCheckpoint, ParseCheckpoint, ScopeSnapshot}; -use crate::incremental::lex::create_lex_checkpoints; -use perl_lexer::{PerlLexer, Token, TokenType}; +use crate::incremental::lex::lex_source_with_checkpoints; +use perl_lexer::Token; use perl_line_index::LineIndex; use perl_parser_core::ast::{Node, NodeKind}; use perl_parser_core::error::ParseOutput; @@ -11,6 +11,11 @@ use ropey::Rope; pub struct IncrementalState { pub rope: Rope, pub line_index: LineIndex, + /// Compact compatibility summaries of live lexer restart boundaries. + /// + /// These summaries are captured from `PerlLexer::checkpoint()` while + /// lexing. The full live state is replayed and validated before any restart; + /// this summary alone never authorizes restoration. pub lex_checkpoints: Vec, pub parse_checkpoints: Vec, /// Authoritative native parser output for the current source. @@ -39,24 +44,16 @@ impl IncrementalState { let mut parser = Parser::new(&source); let parse_output = parser.parse_with_recovery(); let ast = parse_output.ast.clone(); - let mut lexer = PerlLexer::new(&source); - let mut tokens = Vec::new(); - while let Some(token) = lexer.next_token() { - if token.token_type == TokenType::EOF { - break; - } - tokens.push(token); - } - let lex_checkpoints = create_lex_checkpoints(&tokens, &line_index); + let lexed = lex_source_with_checkpoints(&source, &line_index); let parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast); Self { rope, line_index, - lex_checkpoints, + lex_checkpoints: lexed.checkpoints, parse_checkpoints, parse_output, ast, - tokens, + tokens: lexed.tokens, source, } } From fdaf550088f8366600980c5592c19426cddea3b7 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:12:31 -0400 Subject: [PATCH 10/48] fix(parser): relex live checkpoint suffix to EOF --- crates/perl-parser/src/incremental/reparse.rs | 259 +++++++++--------- 1 file changed, 128 insertions(+), 131 deletions(-) diff --git a/crates/perl-parser/src/incremental/reparse.rs b/crates/perl-parser/src/incremental/reparse.rs index 6ec03b595a..f1cce97816 100644 --- a/crates/perl-parser/src/incremental/reparse.rs +++ b/crates/perl-parser/src/incremental/reparse.rs @@ -1,27 +1,19 @@ use crate::incremental::{ - IncrementalState, diagnostics::ReparseResult, edit::Edit, lex::create_lex_checkpoints, + IncrementalState, + diagnostics::{LexRestartReport, LexRestartStrategy, ReparseResult}, + edit::Edit, + lex::{capture_live_checkpoint, lex_from_live_checkpoint, lex_source_with_checkpoints}, }; use anyhow::Result; -use perl_lexer::{PerlLexer, TokenType}; use ropey::Rope; use std::ops::Range; pub(crate) struct SingleEditReparse { pub(crate) range: Range, - pub(crate) reused_tokens: usize, + pub(crate) lex_restart: LexRestartReport, pub(crate) token_count: usize, } -/// Apply a (possibly negative) `byte_shift` to a token offset without wrapping. -/// -/// A negative `byte_shift` whose magnitude exceeds `offset` would underflow when -/// `(offset as isize + byte_shift)` is cast back to `usize`, silently wrapping to -/// a huge value and corrupting the token stream. Clamp the result to 0 instead so -/// the shifted offset stays valid (fixes #2471). -fn shift_offset(offset: usize, byte_shift: isize) -> usize { - (offset as isize).saturating_add(byte_shift).max(0) as usize -} - pub(crate) fn apply_text_edit_to_state(state: &mut IncrementalState, edit: &Edit) -> Result<()> { let old_end = edit.old_end_byte.min(state.source.len()); let start = edit.start_byte.min(state.source.len()); @@ -45,135 +37,145 @@ pub(crate) fn apply_single_edit( state: &mut IncrementalState, edit: &Edit, ) -> Result { - let Some(cp) = state.find_lex_checkpoint(edit.start_byte).copied() else { + let Some(summary) = state.find_lex_checkpoint(edit.start_byte).copied() else { apply_text_edit_to_state(state, edit)?; - anyhow::bail!("No checkpoint found"); + anyhow::bail!("No lexer restart boundary found"); }; - let old_end = edit.old_end_byte.min(state.source.len()); - let start = edit.start_byte.min(state.source.len()); - let byte_shift = edit.new_text.len() as isize - (old_end - start) as isize; + let Some(live_checkpoint) = capture_live_checkpoint(&state.source, summary.byte) else { + apply_text_edit_to_state(state, edit)?; + anyhow::bail!("Could not reproduce complete live lexer state at restart boundary"); + }; + + let restart_byte = live_checkpoint.position; + let reused_prefix_tokens = + state.tokens.iter().take_while(|token| token.start < restart_byte).count(); apply_text_edit_to_state(state, edit)?; - use perl_lexer::{Checkpointable, LexerCheckpoint, Position}; - let mut lexer = PerlLexer::new(&state.source); - let mut lex_cp = LexerCheckpoint::new(); - lex_cp.position = cp.byte; - lex_cp.mode = cp.mode; - lex_cp.current_pos = - Position { byte: cp.byte, line: (cp.line + 1) as u32, column: (cp.column + 1) as u32 }; - lexer.restore(&lex_cp); - let start_idx = - state.tokens.iter().position(|t| t.start >= cp.byte).unwrap_or(state.tokens.len()); - let edit_end_in_new = start + edit.new_text.len(); - let old_sync_start = - state.tokens.iter().position(|t| t.start >= old_end).unwrap_or(state.tokens.len()); - let mut new_tokens = Vec::new(); - let mut last = cp.byte; - let mut synced = false; - let mut sync_old_idx = state.tokens.len(); - while let Some(token) = lexer.next_token() { - if token.token_type == TokenType::EOF { - break; - } - if token.end <= last { - anyhow::bail!("incremental lexer did not advance at byte {}", token.start); - } - last = token.end; - if token.start >= edit_end_in_new { - let mut found = false; - for (off, old_tok) in state.tokens[old_sync_start..].iter().enumerate() { - let shifted_start = shift_offset(old_tok.start, byte_shift); - let shifted_end = shift_offset(old_tok.end, byte_shift); - if token.start == shifted_start - && token.end == shifted_end - && token.token_type == old_tok.token_type - { - found = true; - sync_old_idx = old_sync_start + off + 1; - break; - } - } - new_tokens.push(token); - if found { - synced = true; - break; - } - } else { - new_tokens.push(token); - } - } - let reused_tokens = if synced { state.tokens.len().saturating_sub(sync_old_idx) } else { 0 }; - if synced { - for old_tok in &state.tokens[sync_old_idx..] { - let mut adjusted = old_tok.clone(); - adjusted.start = shift_offset(adjusted.start, byte_shift); - adjusted.end = shift_offset(adjusted.end, byte_shift); - last = adjusted.end; - new_tokens.push(adjusted); - } - } - state.tokens.splice(start_idx.., new_tokens); - state.lex_checkpoints = create_lex_checkpoints(&state.tokens, &state.line_index); - Ok(SingleEditReparse { range: cp.byte..last, reused_tokens, token_count: state.tokens.len() }) + let lexed = + lex_from_live_checkpoint(&state.source, &state.line_index, &live_checkpoint)?; + + state.tokens.truncate(reused_prefix_tokens); + state.tokens.extend(lexed.tokens); + + let mut checkpoints = state + .lex_checkpoints + .iter() + .take_while(|checkpoint| checkpoint.byte < restart_byte) + .copied() + .collect::>(); + checkpoints.extend(lexed.checkpoints); + state.lex_checkpoints = checkpoints; + + let lex_restart = LexRestartReport { + strategy: LexRestartStrategy::LiveCheckpointToEof, + restart_byte, + relexed_bytes: state.source.len().saturating_sub(restart_byte), + reused_prefix_tokens, + reused_suffix_tokens: 0, + }; + + Ok(SingleEditReparse { + range: restart_byte..state.source.len(), + lex_restart, + token_count: state.tokens.len(), + }) } pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result { state.refresh_parse_output(); - let mut lexer = PerlLexer::new(&state.source); - let mut tokens = Vec::new(); - while let Some(token) = lexer.next_token() { - if token.token_type == TokenType::EOF { - break; - } - tokens.push(token); - } - state.tokens = tokens; state.rope = Rope::from_str(&state.source); state.line_index = perl_line_index::LineIndex::new(&state.source); - state.lex_checkpoints = create_lex_checkpoints(&state.tokens, &state.line_index); + let lexed = lex_source_with_checkpoints(&state.source, &state.line_index); + state.tokens = lexed.tokens; + state.lex_checkpoints = lexed.checkpoints; + + let lex_restart = LexRestartReport { + strategy: LexRestartStrategy::FullRelex, + restart_byte: 0, + relexed_bytes: state.source.len(), + reused_prefix_tokens: 0, + reused_suffix_tokens: 0, + }; + Ok(ReparseResult { changed_ranges: vec![0..state.source.len()], parse_output: state.parse_output.clone(), diagnostics: vec![], + lex_restart, reparsed_bytes: state.source.len(), - reused_tokens: 0, + reused_tokens: lex_restart.reused_tokens(), token_count: state.tokens.len(), }) } #[cfg(test)] -mod reparse_offset_tests { +mod tests { use super::*; - use crate::incremental::IncrementalState; - use anyhow::Result; + use perl_lexer::{PerlLexer, Token, TokenType}; + + fn fresh_tokens(source: &str) -> Vec { + let mut lexer = PerlLexer::new(source); + let mut tokens = Vec::new(); + while let Some(token) = lexer.next_token() { + if token.token_type == TokenType::EOF { + break; + } + tokens.push(token); + } + tokens + } + + fn assert_tokens_equal(actual: &[Token], expected: &[Token]) { + assert_eq!(actual.len(), expected.len(), "token count diverged"); + for (index, (actual, expected)) in actual.iter().zip(expected).enumerate() { + assert_eq!(actual.token_type, expected.token_type, "token kind {index}"); + assert_eq!(actual.text, expected.text, "token payload {index}"); + assert_eq!(actual.start, expected.start, "token start {index}"); + assert_eq!(actual.end, expected.end, "token end {index}"); + } + } #[test] - fn shift_offset_clamps_negative_underflow_to_zero() { - // A negative `byte_shift` whose magnitude exceeds the offset previously - // wrapped to a huge usize via `(offset as isize + byte_shift) as usize`. - // It must clamp to 0 instead (regression for #2471). - assert_eq!(shift_offset(3, -10), 0); - assert_eq!(shift_offset(0, -1), 0); - // Exact boundary: shifting to zero is fine, one past zero clamps. - assert_eq!(shift_offset(5, -5), 0); - assert_eq!(shift_offset(5, -6), 0); - // Non-underflowing shifts are unaffected. - assert_eq!(shift_offset(5, -2), 3); - assert_eq!(shift_offset(5, 0), 5); - assert_eq!(shift_offset(5, 4), 9); + fn equal_width_edit_relexes_to_eof_without_speculative_suffix_reuse() -> Result<()> { + let source = "my $x = 1; my $y = 2;"; + let start = source.find("= 1").ok_or_else(|| anyhow::anyhow!("literal missing"))? + 2; + let edit = Edit { + start_byte: start, + old_end_byte: start + 1, + new_end_byte: start + 1, + new_text: "9".to_string(), + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_single_edit(&mut state, &edit)?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::LiveCheckpointToEof); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_eq!(result.range.end, state.source.len()); + assert_tokens_equal(&state.tokens, &fresh_tokens(&state.source)); + Ok(()) + } + + #[test] + fn method_name_edit_restores_after_arrow_state_and_matches_fresh_lex() -> Result<()> { + let source = "$object->method(); my $x = 1;"; + let start = source.find("method").ok_or_else(|| anyhow::anyhow!("method missing"))?; + let edit = Edit { + start_byte: start, + old_end_byte: start + "method".len(), + new_end_byte: start + "member".len(), + new_text: "member".to_string(), + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_single_edit(&mut state, &edit)?; + + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_tokens_equal(&state.tokens, &fresh_tokens(&state.source)); + Ok(()) } #[test] - fn apply_single_edit_negative_shift_does_not_wrap_token_offsets() -> Result<()> { - // Build a document whose tokens near the start sit at small byte offsets, - // then delete a large run from the very beginning so `byte_shift` is - // negative and larger in magnitude than those token positions. Before the - // fix, the reused/sync token offsets wrapped to enormous usize values; now - // they must stay bounded by the (much smaller) new source length. - let source = "my $a = 1;\nmy $b = 2;\nmy $c = 3;\nmy $d = 4;\n".to_string(); - let mut state = IncrementalState::new(source.clone()); - - // Delete the first statement plus newline (12 bytes) from offset 0. + fn length_changing_edit_keeps_every_token_span_in_current_source() -> Result<()> { + let source = "my $a = 1;\nmy $b = 2;\nmy $c = 3;\n"; let delete_len = "my $a = 1;\n".len(); let edit = Edit { start_byte: 0, @@ -181,22 +183,17 @@ mod reparse_offset_tests { new_end_byte: 0, new_text: String::new(), }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_single_edit(&mut state, &edit)?; - // `apply_single_edit` may legitimately bail (e.g. no usable checkpoint), - // in which case the wrapping branch was never reached. The invariant we - // assert is: if it succeeds, every resulting token offset is in range. - if apply_single_edit(&mut state, &edit).is_ok() { - let new_len = state.source.len(); - for tok in &state.tokens { - assert!( - tok.start <= new_len && tok.end <= new_len, - "token offset wrapped: start={}, end={}, source len={}", - tok.start, - tok.end, - new_len, - ); - assert!(tok.start <= tok.end, "token start exceeds end: {tok:?}"); - } + assert_eq!(result.lex_restart.restart_byte, 0); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_tokens_equal(&state.tokens, &fresh_tokens(&state.source)); + for token in &state.tokens { + assert!(token.start <= token.end); + assert!(token.end <= state.source.len()); + assert!(state.source.is_char_boundary(token.start)); + assert!(state.source.is_char_boundary(token.end)); } Ok(()) } From 69f120e10f937e136244ba82fd4415e356293f41 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:13:28 -0400 Subject: [PATCH 11/48] fix(parser): return truthful lexer restart receipt --- crates/perl-parser/src/incremental/mod.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/perl-parser/src/incremental/mod.rs b/crates/perl-parser/src/incremental/mod.rs index ec5e466163..081d526e27 100644 --- a/crates/perl-parser/src/incremental/mod.rs +++ b/crates/perl-parser/src/incremental/mod.rs @@ -15,7 +15,7 @@ use anyhow::Result; pub use perl_line_index::LineIndex; pub use checkpoint::{LexCheckpoint, ParseCheckpoint, ScopeSnapshot}; -pub use diagnostics::ReparseResult; +pub use diagnostics::{LexRestartReport, LexRestartStrategy, ReparseResult}; pub use edit::Edit; use reparse::{apply_single_edit, apply_text_edit_to_state, full_reparse}; pub use state::IncrementalState; @@ -56,19 +56,20 @@ pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result reparse, Err(_) => return full_reparse(state), }; - let reparsed_bytes = reparse.range.end - reparse.range.start; - // The token fast path does not define a second parser-output contract. - // Refresh from the same recovery-aware parser entry point used by a - // fresh parse, then return that exact current-generation output. + // The token restart path does not define a second parser-output + // contract. Refresh through the same recovery-aware full parser used by + // a fresh parse and report that parser work honestly. state.refresh_parse_output(); + let reused_tokens = reparse.lex_restart.reused_tokens(); Ok(ReparseResult { changed_ranges: vec![reparse.range], parse_output: state.parse_output.clone(), diagnostics: vec![], - reparsed_bytes, - reused_tokens: reparse.reused_tokens, + lex_restart: reparse.lex_restart, + reparsed_bytes: state.source.len(), + reused_tokens, token_count: reparse.token_count, }) } else { From 97abf545304c238733122112cbec920798f70e0d Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:14:01 -0400 Subject: [PATCH 12/48] fix(parser): keep one replayable summary per byte boundary --- crates/perl-parser/src/incremental/lex.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/perl-parser/src/incremental/lex.rs b/crates/perl-parser/src/incremental/lex.rs index ef6c84aaad..7ecaced09d 100644 --- a/crates/perl-parser/src/incremental/lex.rs +++ b/crates/perl-parser/src/incremental/lex.rs @@ -20,6 +20,19 @@ fn summarize_checkpoint( LexCheckpoint { byte: checkpoint.position, mode: checkpoint.mode, line, column } } +fn push_summary( + summaries: &mut Vec, + checkpoint: &LiveLexerCheckpoint, + line_index: &LineIndex, +) { + // A queued/virtual lexer event may expose several internal states at one + // byte. The public summary is intentionally one replayable boundary per + // byte and corresponds to the first complete live state at that boundary. + if summaries.last().is_none_or(|summary| summary.byte != checkpoint.position) { + summaries.push(summarize_checkpoint(checkpoint, line_index)); + } +} + /// Lex one complete source and capture restart candidates from the lexer's /// actual live state before every emitted token and before terminal EOF. pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) -> LexedSource { @@ -31,7 +44,7 @@ pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) loop { let live = lexer.checkpoint(); - checkpoints.push(summarize_checkpoint(&live, line_index)); + push_summary(&mut checkpoints, &live, line_index); #[cfg(test)] live_checkpoints.push(live); @@ -101,7 +114,7 @@ pub(crate) fn lex_from_live_checkpoint( loop { let live = lexer.checkpoint(); - checkpoints.push(summarize_checkpoint(&live, line_index)); + push_summary(&mut checkpoints, &live, line_index); #[cfg(test)] live_checkpoints.push(live); From 4cecd1bf77c8c22039942d8772215ce4515bb67c Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:14:32 -0400 Subject: [PATCH 13/48] test(parser): prove live checkpoint restart parity --- .../tests/incremental_lexer_restart.rs | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 crates/perl-parser/tests/incremental_lexer_restart.rs diff --git a/crates/perl-parser/tests/incremental_lexer_restart.rs b/crates/perl-parser/tests/incremental_lexer_restart.rs new file mode 100644 index 0000000000..d41c4cc7f3 --- /dev/null +++ b/crates/perl-parser/tests/incremental_lexer_restart.rs @@ -0,0 +1,98 @@ +#![cfg(feature = "incremental")] +//! Public-contract tests for correctness-first live lexer restart. + +use perl_lexer::{PerlLexer, Token, TokenType}; +use perl_parser::{ + Edit, IncrementalState, LexRestartStrategy, apply_edits, +}; + +type TestResult = Result<(), Box>; + +fn fresh_tokens(source: &str) -> Vec { + let mut lexer = PerlLexer::new(source); + let mut tokens = Vec::new(); + while let Some(token) = lexer.next_token() { + if token.token_type == TokenType::EOF { + break; + } + tokens.push(token); + } + tokens +} + +fn assert_tokens_equal(actual: &[Token], expected: &[Token]) { + assert_eq!(actual.len(), expected.len(), "token count diverged"); + for (index, (actual, expected)) in actual.iter().zip(expected).enumerate() { + assert_eq!(actual.token_type, expected.token_type, "token kind {index}"); + assert_eq!(actual.text, expected.text, "token payload {index}"); + assert_eq!(actual.start, expected.start, "token start {index}"); + assert_eq!(actual.end, expected.end, "token end {index}"); + } +} + +#[test] +fn late_equal_width_edit_retains_prefix_and_relexes_the_complete_suffix() -> TestResult { + let source = "my $before = 1; my $target = 2; my $after = 3;"; + let start = source.find("= 2").ok_or("target literal is missing")? + 2; + let edit = Edit { + start_byte: start, + old_end_byte: start + 1, + new_end_byte: start + 1, + new_text: "9".to_string(), + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::LiveCheckpointToEof); + assert!(result.lex_restart.restart_byte > 0); + assert!(result.lex_restart.reused_prefix_tokens > 0); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_eq!( + result.lex_restart.relexed_bytes, + state.source.len() - result.lex_restart.restart_byte + ); + assert_eq!(result.reused_tokens, result.lex_restart.reused_tokens()); + assert_tokens_equal(&state.tokens, &fresh_tokens(&state.source)); + Ok(()) +} + +#[test] +fn method_context_edit_matches_fresh_lexing_after_complete_state_restore() -> TestResult { + let source = "my $value = $object->method(); my $after = 1;"; + let start = source.find("method").ok_or("method name is missing")?; + let edit = Edit { + start_byte: start, + old_end_byte: start + "method".len(), + new_end_byte: start + "member".len(), + new_text: "member".to_string(), + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::LiveCheckpointToEof); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_tokens_equal(&state.tokens, &fresh_tokens(&state.source)); + Ok(()) +} + +#[test] +fn large_edit_reports_full_relex_instead_of_checkpoint_reuse() -> TestResult { + let source = "my $value = 1;"; + let replacement = "my $value = 2;\n".repeat(80); + let edit = Edit { + start_byte: 0, + old_end_byte: source.len(), + new_end_byte: replacement.len(), + new_text: replacement, + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::FullRelex); + assert_eq!(result.lex_restart.restart_byte, 0); + assert_eq!(result.lex_restart.reused_prefix_tokens, 0); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_eq!(result.lex_restart.relexed_bytes, state.source.len()); + assert_tokens_equal(&state.tokens, &fresh_tokens(&state.source)); + Ok(()) +} From 301e8ce9154e4e0da48669d4df633c2de8165ea6 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:15:10 -0400 Subject: [PATCH 14/48] ci(parser): run live lexer restart parity --- scripts/ci/run_parser_integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/ci/run_parser_integration.py b/scripts/ci/run_parser_integration.py index 70ffdd7db6..dc20db639f 100644 --- a/scripts/ci/run_parser_integration.py +++ b/scripts/ci/run_parser_integration.py @@ -114,6 +114,8 @@ def main() -> int: "incremental_parser_accuracy", "--test", "incremental_parse_output", + "--test", + "incremental_lexer_restart", "--", "--test-threads=4", ] From 93f4559e9690931669383d9e07a7e02d36bbb3df Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:21:44 -0400 Subject: [PATCH 15/48] test(parser): import restart strategy from incremental module --- crates/perl-parser/tests/incremental_lexer_restart.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/perl-parser/tests/incremental_lexer_restart.rs b/crates/perl-parser/tests/incremental_lexer_restart.rs index d41c4cc7f3..77e333a54d 100644 --- a/crates/perl-parser/tests/incremental_lexer_restart.rs +++ b/crates/perl-parser/tests/incremental_lexer_restart.rs @@ -2,9 +2,8 @@ //! Public-contract tests for correctness-first live lexer restart. use perl_lexer::{PerlLexer, Token, TokenType}; -use perl_parser::{ - Edit, IncrementalState, LexRestartStrategy, apply_edits, -}; +use perl_parser::incremental::LexRestartStrategy; +use perl_parser::{Edit, IncrementalState, apply_edits}; type TestResult = Result<(), Box>; From f6cfa8fcc75ad656d7453ace3f794b6b9fd6f9e5 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:23:11 -0400 Subject: [PATCH 16/48] fix(parser): suppress restart boundaries inside pending heredocs --- crates/perl-parser/src/incremental/lex.rs | 64 ++++++++++++++++++++--- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/crates/perl-parser/src/incremental/lex.rs b/crates/perl-parser/src/incremental/lex.rs index 7ecaced09d..b45a748cfb 100644 --- a/crates/perl-parser/src/incremental/lex.rs +++ b/crates/perl-parser/src/incremental/lex.rs @@ -33,18 +33,34 @@ fn push_summary( } } +fn update_pending_heredocs(pending: &mut usize, token: &Token) { + match &token.token_type { + TokenType::HeredocStart => *pending = pending.saturating_add(1), + TokenType::HeredocBody(_) => *pending = pending.saturating_sub(1), + _ => {} + } +} + /// Lex one complete source and capture restart candidates from the lexer's -/// actual live state before every emitted token and before terminal EOF. +/// actual live state before emitted tokens and terminal EOF. +/// +/// The current lexer checkpoint contract does not carry the queued-heredoc +/// collection. Restart summaries are therefore suppressed from a heredoc +/// introducer until its body has been emitted; an edit in that region restarts +/// from the last earlier boundary instead of reconstructing missing state. pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) -> LexedSource { let mut lexer = PerlLexer::new(source); let mut tokens = Vec::new(); let mut checkpoints = Vec::new(); #[cfg(test)] let mut live_checkpoints = Vec::new(); + let mut pending_heredocs = 0usize; loop { let live = lexer.checkpoint(); - push_summary(&mut checkpoints, &live, line_index); + if pending_heredocs == 0 { + push_summary(&mut checkpoints, &live, line_index); + } #[cfg(test)] live_checkpoints.push(live); @@ -54,6 +70,7 @@ pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) if token.token_type == TokenType::EOF { break; } + update_pending_heredocs(&mut pending_heredocs, &token); tokens.push(token); } @@ -66,10 +83,11 @@ pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) } /// Replay the old source to one previously captured token boundary and return -/// the complete live lexer checkpoint for that exact boundary. +/// the complete current `Checkpointable` state for that exact boundary. /// /// The public `LexCheckpoint` remains a compact compatibility summary. Restart -/// correctness is authorized only by this full `Checkpointable` state. +/// correctness is authorized only by the full state returned here. Boundaries +/// inside pending heredocs are never entered into the summary set. pub(crate) fn capture_live_checkpoint( source: &str, boundary: usize, @@ -92,9 +110,9 @@ pub(crate) fn capture_live_checkpoint( } } -/// Restore a complete live checkpoint into the edited source and re-lex from -/// that boundary to EOF. No old suffix is reused in this correctness-first -/// strategy. +/// Restore the complete current checkpoint contract into the edited source and +/// re-lex from that boundary to EOF. No old suffix is reused in this +/// correctness-first strategy. pub(crate) fn lex_from_live_checkpoint( source: &str, line_index: &LineIndex, @@ -111,10 +129,13 @@ pub(crate) fn lex_from_live_checkpoint( #[cfg(test)] let mut live_checkpoints = Vec::new(); let mut last_position = checkpoint.position; + let mut pending_heredocs = 0usize; loop { let live = lexer.checkpoint(); - push_summary(&mut checkpoints, &live, line_index); + if pending_heredocs == 0 { + push_summary(&mut checkpoints, &live, line_index); + } #[cfg(test)] live_checkpoints.push(live); @@ -128,6 +149,7 @@ pub(crate) fn lex_from_live_checkpoint( anyhow::bail!("incremental lexer did not advance at byte {}", token.start); } last_position = token.end; + update_pending_heredocs(&mut pending_heredocs, &token); tokens.push(token); } @@ -193,4 +215,30 @@ mod tests { assert_eq!(&replayed, expected); Ok(()) } + + #[test] + fn restart_summaries_skip_pending_heredoc_interior() -> Result<()> { + let source = "my $value = <= body.end), + "no restart boundary may depend on an unrecorded pending-heredoc queue" + ); + Ok(()) + } } From bc76472da732614b780dd431b2434049678f1d92 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:24:53 -0400 Subject: [PATCH 17/48] test(parser): borrow heredoc token kind in restart guard --- crates/perl-parser/src/incremental/lex.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/perl-parser/src/incremental/lex.rs b/crates/perl-parser/src/incremental/lex.rs index b45a748cfb..76e9743bf6 100644 --- a/crates/perl-parser/src/incremental/lex.rs +++ b/crates/perl-parser/src/incremental/lex.rs @@ -229,7 +229,7 @@ mod tests { let body = lexed .tokens .iter() - .find(|token| matches!(token.token_type, TokenType::HeredocBody(_))) + .find(|token| matches!(&token.token_type, TokenType::HeredocBody(_))) .ok_or_else(|| anyhow::anyhow!("heredoc body token is missing"))?; assert!( From 72e3af30e3df6105f13505131314c61c4e948a41 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:06:34 -0400 Subject: [PATCH 18/48] fix(parser): apply incremental batches atomically --- crates/perl-parser/src/incremental/mod.rs | 122 ++++++++++++++++++---- 1 file changed, 101 insertions(+), 21 deletions(-) diff --git a/crates/perl-parser/src/incremental/mod.rs b/crates/perl-parser/src/incremental/mod.rs index ec5e466163..488e035754 100644 --- a/crates/perl-parser/src/incremental/mod.rs +++ b/crates/perl-parser/src/incremental/mod.rs @@ -32,52 +32,132 @@ pub mod incremental_integration; pub mod incremental_simple; pub mod incremental_v2; +fn validate_edits(source: &str, edits: &[Edit]) -> Result { + let mut by_start = edits.iter().collect::>(); + by_start.sort_by_key(|edit| (edit.start_byte, edit.old_end_byte)); + + let mut previous: Option<&Edit> = None; + let mut total_changed = 0usize; + + for edit in by_start { + if edit.start_byte > edit.old_end_byte || edit.old_end_byte > source.len() { + anyhow::bail!( + "incremental edit range {}..{} is invalid for source length {}", + edit.start_byte, + edit.old_end_byte, + source.len() + ); + } + if !source.is_char_boundary(edit.start_byte) + || !source.is_char_boundary(edit.old_end_byte) + { + anyhow::bail!( + "incremental edit range {}..{} is not on UTF-8 boundaries", + edit.start_byte, + edit.old_end_byte + ); + } + + let expected_new_end = edit + .start_byte + .checked_add(edit.new_text.len()) + .ok_or_else(|| anyhow::anyhow!("incremental edit new range overflows usize"))?; + if edit.new_end_byte != expected_new_end { + anyhow::bail!( + "incremental edit new_end_byte {} does not match replacement end {}", + edit.new_end_byte, + expected_new_end + ); + } + + if let Some(previous) = previous { + if edit.start_byte < previous.old_end_byte || edit.start_byte == previous.start_byte { + anyhow::bail!( + "incremental edit ranges are overlapping or share an ambiguous start: {}..{} and {}..{}", + previous.start_byte, + previous.old_end_byte, + edit.start_byte, + edit.old_end_byte + ); + } + } + previous = Some(edit); + + total_changed = total_changed + .checked_add(edit.touched_bytes()) + .ok_or_else(|| anyhow::anyhow!("incremental edit byte total overflows usize"))?; + } + + Ok(total_changed) +} + +fn apply_text_edits(state: &mut IncrementalState, edits_descending: &[Edit]) -> Result<()> { + for edit in edits_descending { + apply_text_edit_to_state(state, edit)?; + } + Ok(()) +} + +fn full_reparse_after_edits( + state: &mut IncrementalState, + edits_descending: &[Edit], +) -> Result { + let mut candidate = state.clone(); + apply_text_edits(&mut candidate, edits_descending)?; + let result = full_reparse(&mut candidate)?; + *state = candidate; + Ok(result) +} + /// Apply edits incrementally. pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result { + let total_changed = validate_edits(&state.source, edits)?; + + // Edits use coordinates from the same old source generation. Applying them + // from the end preserves every earlier coordinate without offset adjustment. let mut sorted_edits = edits.to_vec(); - sorted_edits.sort_by_key(|e| e.start_byte); + sorted_edits.sort_by_key(|edit| edit.start_byte); sorted_edits.reverse(); - let total_changed = sorted_edits.iter().map(Edit::touched_bytes).sum::(); - if total_changed > MAX_EDIT_SIZE { - return full_reparse(state); + return full_reparse_after_edits(state, &sorted_edits); } if sorted_edits.len() == 1 { let edit = &sorted_edits[0]; if edit.touched_bytes() > 1024 || edit.new_text.matches('\n').count() > 10 { - apply_text_edit_to_state(state, edit)?; - return full_reparse(state); + return full_reparse_after_edits(state, &sorted_edits); } - let reparse = match apply_single_edit(state, edit) { + // Work on a candidate generation. A token-path failure may happen after + // mutating source or token state; no partial generation is published. + let mut candidate = state.clone(); + let reparse = match apply_single_edit(&mut candidate, edit) { Ok(reparse) => reparse, - Err(_) => return full_reparse(state), + Err(_) => return full_reparse_after_edits(state, &sorted_edits), }; - let reparsed_bytes = reparse.range.end - reparse.range.start; // The token fast path does not define a second parser-output contract. // Refresh from the same recovery-aware parser entry point used by a - // fresh parse, then return that exact current-generation output. - state.refresh_parse_output(); - - Ok(ReparseResult { + // fresh parse, then report the complete parser work truthfully. + candidate.refresh_parse_output(); + let reparsed_bytes = candidate.source.len(); + let result = ReparseResult { changed_ranges: vec![reparse.range], - parse_output: state.parse_output.clone(), + parse_output: candidate.parse_output.clone(), diagnostics: vec![], reparsed_bytes, reused_tokens: reparse.reused_tokens, token_count: reparse.token_count, - }) + }; + *state = candidate; + Ok(result) } else { - for edit in sorted_edits { - if apply_single_edit(state, &edit).is_err() { - return full_reparse(state); - } - } - full_reparse(state) + // Multi-edit batches already finish with a complete parser invocation. + // Apply the whole validated batch first rather than publishing a prefix + // when one intermediate token-restart attempt fails. + full_reparse_after_edits(state, &sorted_edits) } } From 874cf908c4a1cad4280315e3a3b986bb676e4089 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:07:14 -0400 Subject: [PATCH 19/48] test(parser): prove atomic full-fallback generations --- .../tests/incremental_parse_output.rs | 97 +++++++++++++++++-- 1 file changed, 89 insertions(+), 8 deletions(-) diff --git a/crates/perl-parser/tests/incremental_parse_output.rs b/crates/perl-parser/tests/incremental_parse_output.rs index 478a7ad6e0..980e48027d 100644 --- a/crates/perl-parser/tests/incremental_parse_output.rs +++ b/crates/perl-parser/tests/incremental_parse_output.rs @@ -1,6 +1,7 @@ #![cfg(feature = "incremental")] //! Differential tests for the incremental native parse-output contract. +use perl_parser::incremental::MAX_EDIT_SIZE; use perl_parser::{Edit, IncrementalState, ParseOutput, Parser, apply_edits}; type TestResult = Result<(), Box>; @@ -28,19 +29,33 @@ fn assert_output_equivalent(actual: &ParseOutput, expected: &ParseOutput) { ); } -fn apply_reference_edit(source: &str, edit: &Edit) -> Result { - if edit.start_byte > edit.old_end_byte || edit.old_end_byte > source.len() { - return Err("reference edit range is out of bounds"); - } - if !source.is_char_boundary(edit.start_byte) || !source.is_char_boundary(edit.old_end_byte) { - return Err("reference edit range is not on UTF-8 boundaries"); - } +fn apply_reference_edits(source: &str, edits: &[Edit]) -> Result { + let mut sorted = edits.to_vec(); + sorted.sort_by_key(|edit| edit.start_byte); + sorted.reverse(); let mut result = source.to_string(); - result.replace_range(edit.start_byte..edit.old_end_byte, &edit.new_text); + for edit in sorted { + if edit.start_byte > edit.old_end_byte || edit.old_end_byte > result.len() { + return Err("reference edit range is out of bounds"); + } + if !result.is_char_boundary(edit.start_byte) + || !result.is_char_boundary(edit.old_end_byte) + { + return Err("reference edit range is not on UTF-8 boundaries"); + } + if edit.new_end_byte != edit.start_byte + edit.new_text.len() { + return Err("reference edit new_end_byte is inconsistent"); + } + result.replace_range(edit.start_byte..edit.old_end_byte, &edit.new_text); + } Ok(result) } +fn apply_reference_edit(source: &str, edit: &Edit) -> Result { + apply_reference_edits(source, std::slice::from_ref(edit)) +} + #[test] fn initial_malformed_state_keeps_the_native_recovered_tree_and_diagnostics() -> TestResult { let source = "my $x = ; print 1;"; @@ -71,6 +86,7 @@ fn clean_to_malformed_edit_returns_the_current_native_parse_output() -> TestResu let result = apply_edits(&mut state, &[edit])?; assert_eq!(state.source, final_source); + assert_eq!(result.reparsed_bytes, final_source.len()); assert_output_equivalent(&state.parse_output, &fresh); assert_output_equivalent(&result.parse_output, &fresh); @@ -96,8 +112,73 @@ fn malformed_to_clean_edit_removes_recovery_diagnostics_atomically() -> TestResu let result = apply_edits(&mut state, &[edit])?; assert_eq!(state.source, final_source); + assert_eq!(result.reparsed_bytes, final_source.len()); assert_output_equivalent(&state.parse_output, &fresh); assert_output_equivalent(&result.parse_output, &fresh); Ok(()) } + +#[test] +fn oversized_batch_applies_every_edit_before_full_fallback() -> TestResult { + let source = "my $left = 1;\nmy $right = 2;\n"; + let second_start = source.find("my $right").ok_or("second statement is missing")?; + let padding = " ".repeat(MAX_EDIT_SIZE / 2 + 1); + let edits = vec![ + Edit { + start_byte: 0, + old_end_byte: 0, + new_end_byte: padding.len(), + new_text: padding.clone(), + }, + Edit { + start_byte: second_start, + old_end_byte: second_start, + new_end_byte: second_start + padding.len(), + new_text: padding, + }, + ]; + let final_source = apply_reference_edits(source, &edits)?; + let fresh = fresh_output(&final_source); + + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &edits)?; + + assert_eq!(state.source, final_source); + assert_eq!(result.changed_ranges, vec![0..final_source.len()]); + assert_eq!(result.reparsed_bytes, final_source.len()); + assert_output_equivalent(&state.parse_output, &fresh); + assert_output_equivalent(&result.parse_output, &fresh); + + Ok(()) +} + +#[test] +fn invalid_overlapping_batch_leaves_the_previous_generation_untouched() -> TestResult { + let source = "my $value = 12;"; + let literal = source.find("12").ok_or("literal is missing")?; + let edits = [ + Edit { + start_byte: literal, + old_end_byte: literal + 2, + new_end_byte: literal + 1, + new_text: "3".to_string(), + }, + Edit { + start_byte: literal + 1, + old_end_byte: literal + 2, + new_end_byte: literal + 2, + new_text: "4".to_string(), + }, + ]; + let before = fresh_output(source); + let mut state = IncrementalState::new(source.to_string()); + + let error = apply_edits(&mut state, &edits).expect_err("overlapping edits must be rejected"); + + assert!(error.to_string().contains("overlapping")); + assert_eq!(state.source, source); + assert_output_equivalent(&state.parse_output, &before); + + Ok(()) +} From 4ff0100ecd581f8e20c5db8c6192b7314609acc2 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:13:28 -0400 Subject: [PATCH 20/48] fix(lexer): checkpoint every mutable replay state --- crates/perl-lexer/src/checkpoint/core.rs | 286 ++++++++++++++++------- 1 file changed, 206 insertions(+), 80 deletions(-) diff --git a/crates/perl-lexer/src/checkpoint/core.rs b/crates/perl-lexer/src/checkpoint/core.rs index 12285efddd..eae002af95 100644 --- a/crates/perl-lexer/src/checkpoint/core.rs +++ b/crates/perl-lexer/src/checkpoint/core.rs @@ -1,76 +1,113 @@ use crate::{LexerMode, Position}; use std::fmt; -/// A checkpoint that captures the complete lexer state +/// Replay-safe representation of one queued heredoc. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PendingHeredocCheckpoint { + /// Heredoc terminator label. + pub label: String, + /// Byte offset where the heredoc body begins. + pub body_start: usize, + /// Whether `<<~` indentation is allowed. + pub allow_indent: bool, +} + +/// Replay-safe representation of an in-progress quote-like operator. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct QuoteOperatorCheckpoint { + /// Operator name such as `q`, `qr`, `s`, or `tr`. + pub operator: String, + /// Opening delimiter. + pub delimiter: char, + /// Byte offset where the operator begins. + pub start_pos: usize, +} + +/// A checkpoint that captures all mutable lexer state needed for token replay. +/// +/// Input references and the wall-clock timeout origin are deliberately not +/// persisted. Restore targets supply the edited input, retain their configured +/// lexer policy, and begin a fresh operation-local timeout budget. #[derive(Debug, Clone, PartialEq)] pub struct LexerCheckpoint { - /// Current position in the input + /// Current position in the input. pub position: usize, - /// Current lexer mode (`ExpectTerm`, `ExpectOperator`, etc.) + /// Current lexer mode (`ExpectTerm`, `ExpectOperator`, etc.). pub mode: LexerMode, - /// Stack for nested delimiters in s{}{} constructs + /// Stack for nested delimiters in `s{}{} ` constructs. pub delimiter_stack: Vec, - /// Whether we're inside prototype parens after 'sub' + /// Whether the lexer is inside prototype parentheses after `sub`. pub in_prototype: bool, - /// Paren depth to track when we exit prototype + /// Parenthesis depth used to detect the end of a prototype. pub prototype_depth: usize, - /// Whether we just saw 'sub' and are waiting for a possible prototype + /// Whether `sub` was just emitted and a prototype may follow. pub after_sub: bool, - /// Whether we just saw '->' (suppresses s/tr/y as substitution) + /// Whether `->` was just emitted, suppressing quote-op interpretation. pub after_arrow: bool, /// Depth of hash-subscript brace nesting. - /// When > 0, suppresses quote-op detection inside hash subscripts/slices. pub hash_brace_depth: usize, - /// Whether the lexer just emitted a complete $var/@var/%var token. - /// Used by the `{` handler to distinguish hash subscript openers from block openers. + /// Whether the lexer just emitted a complete variable token. pub after_var_subscript: bool, - /// Depth of open parentheses (used to guard heredoc vs bitshift disambiguation) + /// Depth of open parentheses used by heredoc/bitshift disambiguation. pub paren_depth: usize, - /// Current position with line/column tracking + /// Current position with line/column tracking. pub current_pos: Position, + /// Whether the previous consumed source unit ended a line. + pub after_newline: bool, + /// Ordered heredoc queue waiting for body consumption. + pub pending_heredocs: Vec, + /// Byte offset of the current physical line start. + pub line_start_offset: usize, + /// Whether heredoc body tokens are emitted instead of consumed virtually. + pub emit_heredoc_body_tokens: bool, + /// In-progress quote-operator metadata, when present. + pub current_quote_op: Option, + /// Whether malformed `qw` constructs use the recovery path. + pub qw_recovery_enabled: bool, /// Whether the terminal EOF token has already been emitted. pub eof_emitted: bool, - /// Additional context for complex states + /// Additional context for complex states. pub context: CheckpointContext, } -/// Additional context that may be needed for certain lexer states +/// Additional context that may be needed for certain lexer states. #[derive(Debug, Clone, PartialEq)] pub enum CheckpointContext { - /// Normal lexing + /// Normal lexing. Normal, - /// Inside a heredoc (tracks the terminator) + /// Inside a heredoc. Heredoc { - /// The terminator label (e.g. `END` in `<, }, - /// Inside a quote-like operator + /// Inside a quote-like operator. QuoteLike { - /// The operator name (e.g. `q`, `qq`, `qw`) + /// Operator name such as `q`, `qq`, or `qw`. operator: String, - /// The delimiter character (e.g. `(` in `qw(...)`) + /// Opening delimiter. delimiter: char, - /// Whether the delimiter is a paired bracket (e.g. `(` / `)`) + /// Whether the delimiter is paired. is_paired: bool, }, } impl LexerCheckpoint { - /// Create a new checkpoint with default values + /// Create a checkpoint with the default lexer state. + #[must_use] pub fn new() -> Self { Self { position: 0, @@ -84,22 +121,31 @@ impl LexerCheckpoint { after_var_subscript: false, paren_depth: 0, current_pos: Position::start(), + after_newline: true, + pending_heredocs: Vec::new(), + line_start_offset: 0, + emit_heredoc_body_tokens: false, + current_quote_op: None, + qw_recovery_enabled: true, eof_emitted: false, context: CheckpointContext::Normal, } } - /// Create a checkpoint at a specific position + /// Create a default-state checkpoint at a specific position. + #[must_use] pub fn at_position(position: usize) -> Self { Self { position, ..Self::new() } } - /// Check if this checkpoint is at the start of input + /// Check whether this checkpoint is at the start of input. + #[must_use] pub fn is_at_start(&self) -> bool { self.position == 0 } - /// Calculate the difference between two checkpoints + /// Calculate the difference between two checkpoints. + #[must_use] pub fn diff(&self, other: &Self) -> super::CheckpointDiff { super::CheckpointDiff { position_delta: self.position as isize - other.position as isize, @@ -113,59 +159,136 @@ impl LexerCheckpoint { || self.after_var_subscript != other.after_var_subscript || self.paren_depth != other.paren_depth, eof_state_changed: self.eof_emitted != other.eof_emitted, - context_changed: self.context != other.context, + context_changed: self.context != other.context + || self.after_newline != other.after_newline + || self.pending_heredocs != other.pending_heredocs + || self.line_start_offset != other.line_start_offset + || self.emit_heredoc_body_tokens != other.emit_heredoc_body_tokens + || self.current_quote_op != other.current_quote_op + || self.qw_recovery_enabled != other.qw_recovery_enabled, } } - /// Apply an edit to this checkpoint. + /// Apply an edit to source-relative checkpoint offsets. /// - /// # Behavior - /// - /// * Edit before the checkpoint and ending strictly before it: the byte - /// `position` is shifted by `new_len - old_len`. The `current_pos` - /// line/column tracker is reset to `Position::start()` because we - /// cannot recompute line/column without rescanning the input. - /// * Edit overlapping the checkpoint: the checkpoint is invalidated -- - /// `position` is rewound to `start`, lexer mode and stacks are reset to - /// defaults, and `current_pos` is reset to `Position::start()`. - /// * Edit at or after the checkpoint: no change. - /// - /// `current_pos` is intentionally reset in both the "shifted" and - /// "invalidated" branches so callers always observe a known sentinel value - /// and must rescan from `position` to recover accurate line/column data. + /// An edit overlapping the replay position or another required state offset + /// invalidates the checkpoint and rewinds it to `start`. Offsets after the + /// replaced range are shifted. An edit beginning exactly at an offset leaves + /// that offset anchored before the replacement so the new text is re-lexed. pub fn apply_edit(&mut self, start: usize, old_len: usize, new_len: usize) { - if self.position > start { - if self.position >= start.saturating_add(old_len) { - self.position = self.position.saturating_sub(old_len).saturating_add(new_len); - if let CheckpointContext::Format { start_position } = &mut self.context { - *start_position = self.position; - } - self.current_pos = Position::start(); - self.eof_emitted = false; - } else { - self.position = start; - self.current_pos = Position::start(); - self.eof_emitted = false; - self.mode = LexerMode::ExpectTerm; - self.delimiter_stack.clear(); - self.in_prototype = false; - self.prototype_depth = 0; - self.after_sub = false; - self.after_arrow = false; - self.hash_brace_depth = 0; - self.after_var_subscript = false; - self.paren_depth = 0; - self.context = CheckpointContext::Normal; + let original_position = self.position; + let Some(position) = transform_offset(self.position, start, old_len, new_len) else { + self.invalidate_at(start); + return; + }; + let Some(line_start_offset) = + transform_offset(self.line_start_offset, start, old_len, new_len) + else { + self.invalidate_at(start); + return; + }; + + let mut pending_heredocs = self.pending_heredocs.clone(); + for pending in &mut pending_heredocs { + let Some(body_start) = transform_offset(pending.body_start, start, old_len, new_len) + else { + self.invalidate_at(start); + return; + }; + pending.body_start = body_start; + } + + let mut current_quote_op = self.current_quote_op.clone(); + if let Some(quote) = &mut current_quote_op { + let Some(start_pos) = transform_offset(quote.start_pos, start, old_len, new_len) else { + self.invalidate_at(start); + return; + }; + quote.start_pos = start_pos; + } + + let mut context = self.context.clone(); + let context_valid = match &mut context { + CheckpointContext::Format { start_position } => { + transform_offset(*start_position, start, old_len, new_len) + .map(|shifted| *start_position = shifted) + .is_some() } + CheckpointContext::Regex { flags_position, .. } => flags_position.as_mut().is_none_or( + |flags| { + transform_offset(*flags, start, old_len, new_len) + .map(|shifted| *flags = shifted) + .is_some() + }, + ), + CheckpointContext::Normal + | CheckpointContext::Heredoc { .. } + | CheckpointContext::QuoteLike { .. } => true, + }; + if !context_valid { + self.invalidate_at(start); + return; + } + + self.position = position; + self.line_start_offset = line_start_offset; + self.pending_heredocs = pending_heredocs; + self.current_quote_op = current_quote_op; + self.context = context; + self.eof_emitted = false; + if self.position != original_position { + self.current_pos = Position::start(); } } - /// Validate that this checkpoint is valid for the given input + /// Validate all source-relative checkpoint offsets for an input. + #[must_use] pub fn is_valid_for(&self, input: &str) -> bool { - self.position <= input.len() + offset_is_valid(input, self.position) + && offset_is_valid(input, self.line_start_offset) + && self.line_start_offset <= self.position + && self.pending_heredocs.iter().all(|pending| { + offset_is_valid(input, pending.body_start) + && pending.body_start >= self.line_start_offset + }) + && self.current_quote_op.as_ref().is_none_or(|quote| { + offset_is_valid(input, quote.start_pos) && quote.start_pos <= self.position + }) + && match &self.context { + CheckpointContext::Format { start_position } => { + offset_is_valid(input, *start_position) + } + CheckpointContext::Regex { flags_position, .. } => { + flags_position.is_none_or(|position| offset_is_valid(input, position)) + } + CheckpointContext::Normal + | CheckpointContext::Heredoc { .. } + | CheckpointContext::QuoteLike { .. } => true, + } + } + + fn invalidate_at(&mut self, start: usize) { + let mut reset = Self::new(); + reset.position = start; + *self = reset; } } +fn transform_offset(offset: usize, start: usize, old_len: usize, new_len: usize) -> Option { + let old_end = start.saturating_add(old_len); + if offset <= start { + Some(offset) + } else if offset >= old_end { + Some(offset.saturating_sub(old_len).saturating_add(new_len)) + } else { + None + } +} + +fn offset_is_valid(input: &str, offset: usize) -> bool { + offset <= input.len() && input.is_char_boundary(offset) +} + impl Default for LexerCheckpoint { fn default() -> Self { Self::new() @@ -173,27 +296,30 @@ impl Default for LexerCheckpoint { } impl fmt::Display for LexerCheckpoint { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { write!( - f, - "Checkpoint@{} mode={:?} delims={} proto={} after_sub={}", + formatter, + "Checkpoint@{} mode={:?} delims={} proto={} after_sub={} heredocs={}", self.position, self.mode, self.delimiter_stack.len(), self.in_prototype, - self.after_sub + self.after_sub, + self.pending_heredocs.len() ) } } -/// Trait for types that support checkpointing +/// Trait for lexers that support state checkpointing. pub trait Checkpointable { - /// Create a checkpoint of the current state + /// Capture all mutable state required to replay tokenization. fn checkpoint(&self) -> LexerCheckpoint; - /// Restore state from a checkpoint + /// Restore mutable replay state into a lexer for the target input. + /// + /// The target lexer retains its configured policy and fresh timeout origin. fn restore(&mut self, checkpoint: &LexerCheckpoint); - /// Check if we can restore to a given checkpoint + /// Check whether every source-relative checkpoint offset is valid. fn can_restore(&self, checkpoint: &LexerCheckpoint) -> bool; } From 7ec726c597464f575b9724ed817a7e56e9a5bc93 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:14:18 -0400 Subject: [PATCH 21/48] fix(lexer): restore complete mutable checkpoint state --- crates/perl-lexer/src/checkpoint_impl.rs | 107 ++++++++++++++++++++--- 1 file changed, 93 insertions(+), 14 deletions(-) diff --git a/crates/perl-lexer/src/checkpoint_impl.rs b/crates/perl-lexer/src/checkpoint_impl.rs index 552979a149..ba436a7ee4 100644 --- a/crates/perl-lexer/src/checkpoint_impl.rs +++ b/crates/perl-lexer/src/checkpoint_impl.rs @@ -1,22 +1,27 @@ -use crate::checkpoint::Checkpointable; +use crate::checkpoint::{ + Checkpointable, PendingHeredocCheckpoint, QuoteOperatorCheckpoint, +}; +use crate::heredoc::HeredocSpec; +use crate::quote_handler::QuoteOperatorInfo; use crate::{LexerCheckpoint, LexerMode, PerlLexer, checkpoint}; +use std::sync::Arc; impl Checkpointable for PerlLexer<'_> { fn checkpoint(&self) -> LexerCheckpoint { use checkpoint::CheckpointContext; - // Determine the checkpoint context based on current state let context = if matches!(self.mode, LexerMode::InFormatBody) { CheckpointContext::Format { // Format bodies are consumed atomically by `next_token`, so a - // checkpoint can observe this mode only at the exact position - // where format-body parsing will begin. + // checkpoint can observe this mode only where body parsing begins. start_position: self.position, } } else if !self.delimiter_stack.is_empty() { - // We're in some kind of quote-like construct CheckpointContext::QuoteLike { - operator: String::new(), // Would need to track this + operator: self + .current_quote_op + .as_ref() + .map_or_else(String::new, |quote| quote.operator.clone()), delimiter: self.delimiter_stack.last().copied().unwrap_or('\0'), is_paired: true, } @@ -36,6 +41,26 @@ impl Checkpointable for PerlLexer<'_> { after_var_subscript: self.after_var_subscript, paren_depth: self.paren_depth, current_pos: self.current_pos, + after_newline: self.after_newline, + pending_heredocs: self + .pending_heredocs + .iter() + .map(|pending| PendingHeredocCheckpoint { + label: pending.label.to_string(), + body_start: pending.body_start, + allow_indent: pending.allow_indent, + }) + .collect(), + line_start_offset: self.line_start_offset, + emit_heredoc_body_tokens: self.emit_heredoc_body_tokens, + current_quote_op: self.current_quote_op.as_ref().map(|quote| { + QuoteOperatorCheckpoint { + operator: quote.operator.clone(), + delimiter: quote.delimiter, + start_pos: quote.start_pos, + } + }), + qw_recovery_enabled: self.qw_recovery_enabled, eof_emitted: self.eof_emitted, context, } @@ -53,21 +78,36 @@ impl Checkpointable for PerlLexer<'_> { self.after_var_subscript = checkpoint.after_var_subscript; self.paren_depth = checkpoint.paren_depth; self.current_pos = checkpoint.current_pos; + self.after_newline = checkpoint.after_newline; + self.pending_heredocs = checkpoint + .pending_heredocs + .iter() + .map(|pending| HeredocSpec { + label: Arc::from(pending.label.as_str()), + body_start: pending.body_start, + allow_indent: pending.allow_indent, + }) + .collect(); + self.line_start_offset = checkpoint.line_start_offset; + self.emit_heredoc_body_tokens = checkpoint.emit_heredoc_body_tokens; + self.current_quote_op = checkpoint.current_quote_op.as_ref().map(|quote| { + QuoteOperatorInfo { + operator: quote.operator.clone(), + delimiter: quote.delimiter, + start_pos: quote.start_pos, + } + }); + self.qw_recovery_enabled = checkpoint.qw_recovery_enabled; self.eof_emitted = checkpoint.eof_emitted; - // Handle special contexts use checkpoint::CheckpointContext; - if let CheckpointContext::Format { .. } = &checkpoint.context { - // Ensure we're in format body mode - if !matches!(self.mode, LexerMode::InFormatBody) { - self.mode = LexerMode::InFormatBody; - } + if matches!(checkpoint.context, CheckpointContext::Format { .. }) { + self.mode = LexerMode::InFormatBody; } } fn can_restore(&self, checkpoint: &LexerCheckpoint) -> bool { - // Can restore if the position is valid for our input - checkpoint.position <= self.input.len() + checkpoint.is_valid_for(self.input) } } @@ -75,6 +115,7 @@ impl Checkpointable for PerlLexer<'_> { mod tests { use super::*; use crate::checkpoint::CheckpointContext; + use crate::Position; type TestResult = std::result::Result<(), String>; @@ -145,4 +186,42 @@ mod tests { } Ok(()) } + + #[test] + fn restore_round_trip_preserves_every_mutable_replay_field() { + let input = "x".repeat(96); + let mut lexer = PerlLexer::new(&input); + lexer.position = 32; + lexer.mode = LexerMode::ExpectOperator; + lexer.delimiter_stack = vec!['{', '(']; + lexer.in_prototype = true; + lexer.prototype_depth = 2; + lexer.after_sub = true; + lexer.after_arrow = true; + lexer.hash_brace_depth = 3; + lexer.after_var_subscript = true; + lexer.paren_depth = 4; + lexer.current_pos = Position { byte: 32, line: 3, column: 5 }; + lexer.after_newline = false; + lexer.pending_heredocs = vec![HeredocSpec { + label: Arc::from("END"), + body_start: 48, + allow_indent: true, + }]; + lexer.line_start_offset = 24; + lexer.emit_heredoc_body_tokens = true; + lexer.current_quote_op = Some(QuoteOperatorInfo { + operator: "s".to_string(), + delimiter: '{', + start_pos: 28, + }); + lexer.qw_recovery_enabled = false; + lexer.eof_emitted = false; + + let expected = lexer.checkpoint(); + let mut restored = PerlLexer::new(&input); + restored.restore(&expected); + + assert_eq!(restored.checkpoint(), expected); + } } From 02a88aab8b7d12637819d384a6f9120240139972 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:14:31 -0400 Subject: [PATCH 22/48] feat(lexer): export complete checkpoint state types --- crates/perl-lexer/src/checkpoint/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/perl-lexer/src/checkpoint/mod.rs b/crates/perl-lexer/src/checkpoint/mod.rs index e1cfd45092..720a6a05bb 100644 --- a/crates/perl-lexer/src/checkpoint/mod.rs +++ b/crates/perl-lexer/src/checkpoint/mod.rs @@ -5,7 +5,10 @@ mod core; mod diff; pub use cache::CheckpointCache; -pub use core::{CheckpointContext, Checkpointable, LexerCheckpoint}; +pub use core::{ + CheckpointContext, Checkpointable, LexerCheckpoint, PendingHeredocCheckpoint, + QuoteOperatorCheckpoint, +}; pub use diff::CheckpointDiff; #[cfg(test)] From ce3d568f975a0f81a12d006d89260c77f3273eef Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:15:23 -0400 Subject: [PATCH 23/48] fix(parser): authorize heredoc restart boundaries from live state --- crates/perl-parser/src/incremental/lex.rs | 77 +++++++++++------------ 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/crates/perl-parser/src/incremental/lex.rs b/crates/perl-parser/src/incremental/lex.rs index 76e9743bf6..e8351d70fc 100644 --- a/crates/perl-parser/src/incremental/lex.rs +++ b/crates/perl-parser/src/incremental/lex.rs @@ -26,41 +26,29 @@ fn push_summary( line_index: &LineIndex, ) { // A queued/virtual lexer event may expose several internal states at one - // byte. The public summary is intentionally one replayable boundary per - // byte and corresponds to the first complete live state at that boundary. + // byte. The public summary is one replayable boundary per byte and maps to + // the first complete live state reproduced by `capture_live_checkpoint`. if summaries.last().is_none_or(|summary| summary.byte != checkpoint.position) { summaries.push(summarize_checkpoint(checkpoint, line_index)); } } -fn update_pending_heredocs(pending: &mut usize, token: &Token) { - match &token.token_type { - TokenType::HeredocStart => *pending = pending.saturating_add(1), - TokenType::HeredocBody(_) => *pending = pending.saturating_sub(1), - _ => {} - } -} - /// Lex one complete source and capture restart candidates from the lexer's -/// actual live state before emitted tokens and terminal EOF. +/// actual mutable state before emitted tokens and terminal EOF. /// -/// The current lexer checkpoint contract does not carry the queued-heredoc -/// collection. Restart summaries are therefore suppressed from a heredoc -/// introducer until its body has been emitted; an edit in that region restarts -/// from the last earlier boundary instead of reconstructing missing state. +/// The live checkpoint includes the ordered heredoc queue, newline/line-start +/// context, quote-operator state, body-emission policy, and recovery policy, so +/// heredoc boundaries no longer need a parser-side suppression approximation. pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) -> LexedSource { let mut lexer = PerlLexer::new(source); let mut tokens = Vec::new(); let mut checkpoints = Vec::new(); #[cfg(test)] let mut live_checkpoints = Vec::new(); - let mut pending_heredocs = 0usize; loop { let live = lexer.checkpoint(); - if pending_heredocs == 0 { - push_summary(&mut checkpoints, &live, line_index); - } + push_summary(&mut checkpoints, &live, line_index); #[cfg(test)] live_checkpoints.push(live); @@ -70,7 +58,6 @@ pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) if token.token_type == TokenType::EOF { break; } - update_pending_heredocs(&mut pending_heredocs, &token); tokens.push(token); } @@ -86,8 +73,7 @@ pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) /// the complete current `Checkpointable` state for that exact boundary. /// /// The public `LexCheckpoint` remains a compact compatibility summary. Restart -/// correctness is authorized only by the full state returned here. Boundaries -/// inside pending heredocs are never entered into the summary set. +/// correctness is authorized only by the full state returned here. pub(crate) fn capture_live_checkpoint( source: &str, boundary: usize, @@ -110,7 +96,7 @@ pub(crate) fn capture_live_checkpoint( } } -/// Restore the complete current checkpoint contract into the edited source and +/// Restore the complete mutable checkpoint contract into the edited source and /// re-lex from that boundary to EOF. No old suffix is reused in this /// correctness-first strategy. pub(crate) fn lex_from_live_checkpoint( @@ -129,13 +115,10 @@ pub(crate) fn lex_from_live_checkpoint( #[cfg(test)] let mut live_checkpoints = Vec::new(); let mut last_position = checkpoint.position; - let mut pending_heredocs = 0usize; loop { let live = lexer.checkpoint(); - if pending_heredocs == 0 { - push_summary(&mut checkpoints, &live, line_index); - } + push_summary(&mut checkpoints, &live, line_index); #[cfg(test)] live_checkpoints.push(live); @@ -149,7 +132,6 @@ pub(crate) fn lex_from_live_checkpoint( anyhow::bail!("incremental lexer did not advance at byte {}", token.start); } last_position = token.end; - update_pending_heredocs(&mut pending_heredocs, &token); tokens.push(token); } @@ -217,27 +199,40 @@ mod tests { } #[test] - fn restart_summaries_skip_pending_heredoc_interior() -> Result<()> { + fn heredoc_queue_is_captured_and_restart_boundaries_resume() -> Result<()> { let source = "my $value = < queued.position && checkpoint.pending_heredocs.is_empty() + }) + .ok_or_else(|| anyhow::anyhow!("checkpoint after heredoc completion is missing"))?; assert!( - lexed - .checkpoints - .iter() - .all(|checkpoint| checkpoint.byte <= start.start || checkpoint.byte >= body.end), - "no restart boundary may depend on an unrecorded pending-heredoc queue" + lexed.checkpoints.iter().any(|summary| summary.byte == resumed.position), + "restart summaries must resume after the heredoc queue drains" ); Ok(()) } From a059e4b90b94982a7af33c50292fed09941f7488 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:16:30 -0400 Subject: [PATCH 24/48] fix(parser): adjust live checkpoints across edits --- crates/perl-parser/src/incremental/reparse.rs | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/perl-parser/src/incremental/reparse.rs b/crates/perl-parser/src/incremental/reparse.rs index f1cce97816..69298ca1ff 100644 --- a/crates/perl-parser/src/incremental/reparse.rs +++ b/crates/perl-parser/src/incremental/reparse.rs @@ -41,18 +41,25 @@ pub(crate) fn apply_single_edit( apply_text_edit_to_state(state, edit)?; anyhow::bail!("No lexer restart boundary found"); }; - let Some(live_checkpoint) = capture_live_checkpoint(&state.source, summary.byte) else { + let Some(mut live_checkpoint) = capture_live_checkpoint(&state.source, summary.byte) else { apply_text_edit_to_state(state, edit)?; anyhow::bail!("Could not reproduce complete live lexer state at restart boundary"); }; + let old_len = edit + .old_end_byte + .checked_sub(edit.start_byte) + .ok_or_else(|| anyhow::anyhow!("edit end precedes edit start"))?; + if !live_checkpoint.try_apply_edit(edit.start_byte, old_len, edit.new_text.len()) { + apply_text_edit_to_state(state, edit)?; + anyhow::bail!("Edit invalidated required live lexer state"); + } let restart_byte = live_checkpoint.position; let reused_prefix_tokens = state.tokens.iter().take_while(|token| token.start < restart_byte).count(); apply_text_edit_to_state(state, edit)?; - let lexed = - lex_from_live_checkpoint(&state.source, &state.line_index, &live_checkpoint)?; + let lexed = lex_from_live_checkpoint(&state.source, &state.line_index, &live_checkpoint)?; state.tokens.truncate(reused_prefix_tokens); state.tokens.extend(lexed.tokens); @@ -173,6 +180,25 @@ mod tests { Ok(()) } + #[test] + fn heredoc_body_edit_restores_the_live_queue_and_matches_fresh_lex() -> Result<()> { + let source = "my $value = < Result<()> { let source = "my $a = 1;\nmy $b = 2;\nmy $c = 3;\n"; From e70d350290de5e6da94af7d996584849da22ce5d Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:17:25 -0400 Subject: [PATCH 25/48] fix(lexer): expose checkpoint edit invalidation --- crates/perl-lexer/src/checkpoint/core.rs | 38 ++++++++++++++++-------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/crates/perl-lexer/src/checkpoint/core.rs b/crates/perl-lexer/src/checkpoint/core.rs index eae002af95..2c86af4485 100644 --- a/crates/perl-lexer/src/checkpoint/core.rs +++ b/crates/perl-lexer/src/checkpoint/core.rs @@ -171,21 +171,32 @@ impl LexerCheckpoint { /// Apply an edit to source-relative checkpoint offsets. /// - /// An edit overlapping the replay position or another required state offset - /// invalidates the checkpoint and rewinds it to `start`. Offsets after the - /// replaced range are shifted. An edit beginning exactly at an offset leaves - /// that offset anchored before the replacement so the new text is re-lexed. + /// Invalidated checkpoints retain the historical behavior of rewinding to + /// `start`. Call [`Self::try_apply_edit`] when the caller must distinguish a + /// transformed checkpoint from a conservative reset. pub fn apply_edit(&mut self, start: usize, old_len: usize, new_len: usize) { + let _ = self.try_apply_edit(start, old_len, new_len); + } + + /// Apply an edit and report whether all required replay state survived. + /// + /// An edit overlapping the replay position or another required state offset + /// invalidates the checkpoint and rewinds it to `start`, returning `false`. + /// Offsets after the replaced range are shifted. An edit beginning exactly + /// at an offset leaves it anchored before the replacement so the new text is + /// re-lexed. + #[must_use] + pub fn try_apply_edit(&mut self, start: usize, old_len: usize, new_len: usize) -> bool { let original_position = self.position; let Some(position) = transform_offset(self.position, start, old_len, new_len) else { self.invalidate_at(start); - return; + return false; }; let Some(line_start_offset) = transform_offset(self.line_start_offset, start, old_len, new_len) else { self.invalidate_at(start); - return; + return false; }; let mut pending_heredocs = self.pending_heredocs.clone(); @@ -193,7 +204,7 @@ impl LexerCheckpoint { let Some(body_start) = transform_offset(pending.body_start, start, old_len, new_len) else { self.invalidate_at(start); - return; + return false; }; pending.body_start = body_start; } @@ -202,7 +213,7 @@ impl LexerCheckpoint { if let Some(quote) = &mut current_quote_op { let Some(start_pos) = transform_offset(quote.start_pos, start, old_len, new_len) else { self.invalidate_at(start); - return; + return false; }; quote.start_pos = start_pos; } @@ -227,7 +238,7 @@ impl LexerCheckpoint { }; if !context_valid { self.invalidate_at(start); - return; + return false; } self.position = position; @@ -239,6 +250,7 @@ impl LexerCheckpoint { if self.position != original_position { self.current_pos = Position::start(); } + true } /// Validate all source-relative checkpoint offsets for an input. @@ -247,10 +259,10 @@ impl LexerCheckpoint { offset_is_valid(input, self.position) && offset_is_valid(input, self.line_start_offset) && self.line_start_offset <= self.position - && self.pending_heredocs.iter().all(|pending| { - offset_is_valid(input, pending.body_start) - && pending.body_start >= self.line_start_offset - }) + && self + .pending_heredocs + .iter() + .all(|pending| offset_is_valid(input, pending.body_start)) && self.current_quote_op.as_ref().is_none_or(|quote| { offset_is_valid(input, quote.start_pos) && quote.start_pos <= self.position }) From c1d3a334f6ceddf8d41e4edd054e2bfef1bda2a3 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:17:52 -0400 Subject: [PATCH 26/48] fix(parser): keep stacked edit batches atomic --- crates/perl-parser/src/incremental/mod.rs | 116 +++++++++++++++++----- 1 file changed, 93 insertions(+), 23 deletions(-) diff --git a/crates/perl-parser/src/incremental/mod.rs b/crates/perl-parser/src/incremental/mod.rs index 081d526e27..914da69b61 100644 --- a/crates/perl-parser/src/incremental/mod.rs +++ b/crates/perl-parser/src/incremental/mod.rs @@ -32,53 +32,123 @@ pub mod incremental_integration; pub mod incremental_simple; pub mod incremental_v2; +fn validate_edits(source: &str, edits: &[Edit]) -> Result { + let mut by_start = edits.iter().collect::>(); + by_start.sort_by_key(|edit| (edit.start_byte, edit.old_end_byte)); + + let mut previous: Option<&Edit> = None; + let mut total_changed = 0usize; + + for edit in by_start { + if edit.start_byte > edit.old_end_byte || edit.old_end_byte > source.len() { + anyhow::bail!( + "incremental edit range {}..{} is invalid for source length {}", + edit.start_byte, + edit.old_end_byte, + source.len() + ); + } + if !source.is_char_boundary(edit.start_byte) + || !source.is_char_boundary(edit.old_end_byte) + { + anyhow::bail!( + "incremental edit range {}..{} is not on UTF-8 boundaries", + edit.start_byte, + edit.old_end_byte + ); + } + + let expected_new_end = edit + .start_byte + .checked_add(edit.new_text.len()) + .ok_or_else(|| anyhow::anyhow!("incremental edit new range overflows usize"))?; + if edit.new_end_byte != expected_new_end { + anyhow::bail!( + "incremental edit new_end_byte {} does not match replacement end {}", + edit.new_end_byte, + expected_new_end + ); + } + + if let Some(previous) = previous { + if edit.start_byte < previous.old_end_byte || edit.start_byte == previous.start_byte { + anyhow::bail!( + "incremental edit ranges are overlapping or share an ambiguous start: {}..{} and {}..{}", + previous.start_byte, + previous.old_end_byte, + edit.start_byte, + edit.old_end_byte + ); + } + } + previous = Some(edit); + + total_changed = total_changed + .checked_add(edit.touched_bytes()) + .ok_or_else(|| anyhow::anyhow!("incremental edit byte total overflows usize"))?; + } + + Ok(total_changed) +} + +fn apply_text_edits(state: &mut IncrementalState, edits_descending: &[Edit]) -> Result<()> { + for edit in edits_descending { + apply_text_edit_to_state(state, edit)?; + } + Ok(()) +} + +fn full_reparse_after_edits( + state: &mut IncrementalState, + edits_descending: &[Edit], +) -> Result { + let mut candidate = state.clone(); + apply_text_edits(&mut candidate, edits_descending)?; + let result = full_reparse(&mut candidate)?; + *state = candidate; + Ok(result) +} + /// Apply edits incrementally. pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result { + let total_changed = validate_edits(&state.source, edits)?; + let mut sorted_edits = edits.to_vec(); - sorted_edits.sort_by_key(|e| e.start_byte); + sorted_edits.sort_by_key(|edit| edit.start_byte); sorted_edits.reverse(); - let total_changed = sorted_edits.iter().map(Edit::touched_bytes).sum::(); - if total_changed > MAX_EDIT_SIZE { - return full_reparse(state); + return full_reparse_after_edits(state, &sorted_edits); } if sorted_edits.len() == 1 { let edit = &sorted_edits[0]; if edit.touched_bytes() > 1024 || edit.new_text.matches('\n').count() > 10 { - apply_text_edit_to_state(state, edit)?; - return full_reparse(state); + return full_reparse_after_edits(state, &sorted_edits); } - let reparse = match apply_single_edit(state, edit) { + let mut candidate = state.clone(); + let reparse = match apply_single_edit(&mut candidate, edit) { Ok(reparse) => reparse, - Err(_) => return full_reparse(state), + Err(_) => return full_reparse_after_edits(state, &sorted_edits), }; - // The token restart path does not define a second parser-output - // contract. Refresh through the same recovery-aware full parser used by - // a fresh parse and report that parser work honestly. - state.refresh_parse_output(); + candidate.refresh_parse_output(); let reused_tokens = reparse.lex_restart.reused_tokens(); - - Ok(ReparseResult { + let result = ReparseResult { changed_ranges: vec![reparse.range], - parse_output: state.parse_output.clone(), + parse_output: candidate.parse_output.clone(), diagnostics: vec![], lex_restart: reparse.lex_restart, - reparsed_bytes: state.source.len(), + reparsed_bytes: candidate.source.len(), reused_tokens, token_count: reparse.token_count, - }) + }; + *state = candidate; + Ok(result) } else { - for edit in sorted_edits { - if apply_single_edit(state, &edit).is_err() { - return full_reparse(state); - } - } - full_reparse(state) + full_reparse_after_edits(state, &sorted_edits) } } From 30b211b8945ead745e08051b9e1e713ea209a782 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:18:28 -0400 Subject: [PATCH 27/48] test(parser): retain atomic parse-output proofs in stack --- .../tests/incremental_parse_output.rs | 97 +++++++++++++++++-- 1 file changed, 89 insertions(+), 8 deletions(-) diff --git a/crates/perl-parser/tests/incremental_parse_output.rs b/crates/perl-parser/tests/incremental_parse_output.rs index 478a7ad6e0..980e48027d 100644 --- a/crates/perl-parser/tests/incremental_parse_output.rs +++ b/crates/perl-parser/tests/incremental_parse_output.rs @@ -1,6 +1,7 @@ #![cfg(feature = "incremental")] //! Differential tests for the incremental native parse-output contract. +use perl_parser::incremental::MAX_EDIT_SIZE; use perl_parser::{Edit, IncrementalState, ParseOutput, Parser, apply_edits}; type TestResult = Result<(), Box>; @@ -28,19 +29,33 @@ fn assert_output_equivalent(actual: &ParseOutput, expected: &ParseOutput) { ); } -fn apply_reference_edit(source: &str, edit: &Edit) -> Result { - if edit.start_byte > edit.old_end_byte || edit.old_end_byte > source.len() { - return Err("reference edit range is out of bounds"); - } - if !source.is_char_boundary(edit.start_byte) || !source.is_char_boundary(edit.old_end_byte) { - return Err("reference edit range is not on UTF-8 boundaries"); - } +fn apply_reference_edits(source: &str, edits: &[Edit]) -> Result { + let mut sorted = edits.to_vec(); + sorted.sort_by_key(|edit| edit.start_byte); + sorted.reverse(); let mut result = source.to_string(); - result.replace_range(edit.start_byte..edit.old_end_byte, &edit.new_text); + for edit in sorted { + if edit.start_byte > edit.old_end_byte || edit.old_end_byte > result.len() { + return Err("reference edit range is out of bounds"); + } + if !result.is_char_boundary(edit.start_byte) + || !result.is_char_boundary(edit.old_end_byte) + { + return Err("reference edit range is not on UTF-8 boundaries"); + } + if edit.new_end_byte != edit.start_byte + edit.new_text.len() { + return Err("reference edit new_end_byte is inconsistent"); + } + result.replace_range(edit.start_byte..edit.old_end_byte, &edit.new_text); + } Ok(result) } +fn apply_reference_edit(source: &str, edit: &Edit) -> Result { + apply_reference_edits(source, std::slice::from_ref(edit)) +} + #[test] fn initial_malformed_state_keeps_the_native_recovered_tree_and_diagnostics() -> TestResult { let source = "my $x = ; print 1;"; @@ -71,6 +86,7 @@ fn clean_to_malformed_edit_returns_the_current_native_parse_output() -> TestResu let result = apply_edits(&mut state, &[edit])?; assert_eq!(state.source, final_source); + assert_eq!(result.reparsed_bytes, final_source.len()); assert_output_equivalent(&state.parse_output, &fresh); assert_output_equivalent(&result.parse_output, &fresh); @@ -96,8 +112,73 @@ fn malformed_to_clean_edit_removes_recovery_diagnostics_atomically() -> TestResu let result = apply_edits(&mut state, &[edit])?; assert_eq!(state.source, final_source); + assert_eq!(result.reparsed_bytes, final_source.len()); assert_output_equivalent(&state.parse_output, &fresh); assert_output_equivalent(&result.parse_output, &fresh); Ok(()) } + +#[test] +fn oversized_batch_applies_every_edit_before_full_fallback() -> TestResult { + let source = "my $left = 1;\nmy $right = 2;\n"; + let second_start = source.find("my $right").ok_or("second statement is missing")?; + let padding = " ".repeat(MAX_EDIT_SIZE / 2 + 1); + let edits = vec![ + Edit { + start_byte: 0, + old_end_byte: 0, + new_end_byte: padding.len(), + new_text: padding.clone(), + }, + Edit { + start_byte: second_start, + old_end_byte: second_start, + new_end_byte: second_start + padding.len(), + new_text: padding, + }, + ]; + let final_source = apply_reference_edits(source, &edits)?; + let fresh = fresh_output(&final_source); + + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &edits)?; + + assert_eq!(state.source, final_source); + assert_eq!(result.changed_ranges, vec![0..final_source.len()]); + assert_eq!(result.reparsed_bytes, final_source.len()); + assert_output_equivalent(&state.parse_output, &fresh); + assert_output_equivalent(&result.parse_output, &fresh); + + Ok(()) +} + +#[test] +fn invalid_overlapping_batch_leaves_the_previous_generation_untouched() -> TestResult { + let source = "my $value = 12;"; + let literal = source.find("12").ok_or("literal is missing")?; + let edits = [ + Edit { + start_byte: literal, + old_end_byte: literal + 2, + new_end_byte: literal + 1, + new_text: "3".to_string(), + }, + Edit { + start_byte: literal + 1, + old_end_byte: literal + 2, + new_end_byte: literal + 2, + new_text: "4".to_string(), + }, + ]; + let before = fresh_output(source); + let mut state = IncrementalState::new(source.to_string()); + + let error = apply_edits(&mut state, &edits).expect_err("overlapping edits must be rejected"); + + assert!(error.to_string().contains("overlapping")); + assert_eq!(state.source, source); + assert_output_equivalent(&state.parse_output, &before); + + Ok(()) +} From 9f5aef685511e517799fdfa62c67d4c38a42aeaf Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:17:25 -0400 Subject: [PATCH 28/48] fix(parser): keep empty edit batches generation-neutral --- crates/perl-parser/src/incremental/mod.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/perl-parser/src/incremental/mod.rs b/crates/perl-parser/src/incremental/mod.rs index 488e035754..b799a00d7d 100644 --- a/crates/perl-parser/src/incremental/mod.rs +++ b/crates/perl-parser/src/incremental/mod.rs @@ -91,6 +91,17 @@ fn validate_edits(source: &str, edits: &[Edit]) -> Result { Ok(total_changed) } +fn unchanged_result(state: &IncrementalState) -> ReparseResult { + ReparseResult { + changed_ranges: Vec::new(), + parse_output: state.parse_output.clone(), + diagnostics: Vec::new(), + reparsed_bytes: 0, + reused_tokens: state.tokens.len(), + token_count: state.tokens.len(), + } +} + fn apply_text_edits(state: &mut IncrementalState, edits_descending: &[Edit]) -> Result<()> { for edit in edits_descending { apply_text_edit_to_state(state, edit)?; @@ -112,6 +123,9 @@ fn full_reparse_after_edits( /// Apply edits incrementally. pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result { let total_changed = validate_edits(&state.source, edits)?; + if edits.is_empty() { + return Ok(unchanged_result(state)); + } // Edits use coordinates from the same old source generation. Applying them // from the end preserves every earlier coordinate without offset adjustment. From e2d9b7ae5155970f0a16f0d0c217f54c599cc8c8 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:18:01 -0400 Subject: [PATCH 29/48] test(parser): prove empty edit batches are no-ops --- .../tests/incremental_parse_output.rs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/perl-parser/tests/incremental_parse_output.rs b/crates/perl-parser/tests/incremental_parse_output.rs index 980e48027d..6dcba3a737 100644 --- a/crates/perl-parser/tests/incremental_parse_output.rs +++ b/crates/perl-parser/tests/incremental_parse_output.rs @@ -68,6 +68,28 @@ fn initial_malformed_state_keeps_the_native_recovered_tree_and_diagnostics() -> Ok(()) } +#[test] +fn empty_edit_batch_preserves_the_current_generation_without_parser_work() -> TestResult { + let source = "my $x = ; print 1;"; + let mut state = IncrementalState::new(source.to_string()); + let before = state.parse_output.clone(); + let token_count = state.tokens.len(); + assert!(!before.diagnostics.is_empty(), "fixture must preserve recovered output"); + + let result = apply_edits(&mut state, &[])?; + + assert_eq!(state.source, source); + assert!(result.changed_ranges.is_empty()); + assert_eq!(result.reparsed_bytes, 0); + assert_eq!(result.reused_tokens, token_count); + assert_eq!(result.token_count, token_count); + assert_eq!(state.tokens.len(), token_count); + assert_output_equivalent(&state.parse_output, &before); + assert_output_equivalent(&result.parse_output, &before); + + Ok(()) +} + #[test] fn clean_to_malformed_edit_returns_the_current_native_parse_output() -> TestResult { let source = "my $x = 1; print 2;"; From c6a7b8623b115f0d581a6e14ec1ca76dc21e2230 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:20:33 -0400 Subject: [PATCH 30/48] feat(parser): model unchanged lexer restart receipts --- crates/perl-parser/src/incremental/diagnostics.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/perl-parser/src/incremental/diagnostics.rs b/crates/perl-parser/src/incremental/diagnostics.rs index 2fc356a410..18b9d062a3 100644 --- a/crates/perl-parser/src/incremental/diagnostics.rs +++ b/crates/perl-parser/src/incremental/diagnostics.rs @@ -6,6 +6,8 @@ use std::ops::Range; #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum LexRestartStrategy { + /// Reuse the current token stream without performing lexer work. + Unchanged, /// Lex the complete current source from byte zero. FullRelex, /// Restore one complete live lexer checkpoint and re-lex from there to EOF. @@ -19,6 +21,9 @@ pub struct LexRestartReport { /// Strategy that produced the current token stream. pub strategy: LexRestartStrategy, /// Byte boundary where fresh lexing began. + /// + /// For [`LexRestartStrategy::Unchanged`], this is the current source length: + /// the complete old token stream is retained and no byte is freshly lexed. pub restart_byte: usize, /// Number of source bytes lexed from the restart boundary to EOF. pub relexed_bytes: usize, From d48d6d1177ff86f1da242eca1134d2bc37e8853f Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:21:10 -0400 Subject: [PATCH 31/48] fix(parser): report empty edit batches as unchanged --- crates/perl-parser/src/incremental/mod.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/perl-parser/src/incremental/mod.rs b/crates/perl-parser/src/incremental/mod.rs index 914da69b61..7ff2cafce4 100644 --- a/crates/perl-parser/src/incremental/mod.rs +++ b/crates/perl-parser/src/incremental/mod.rs @@ -91,6 +91,25 @@ fn validate_edits(source: &str, edits: &[Edit]) -> Result { Ok(total_changed) } +fn unchanged_result(state: &IncrementalState) -> ReparseResult { + let lex_restart = LexRestartReport { + strategy: LexRestartStrategy::Unchanged, + restart_byte: state.source.len(), + relexed_bytes: 0, + reused_prefix_tokens: state.tokens.len(), + reused_suffix_tokens: 0, + }; + ReparseResult { + changed_ranges: Vec::new(), + parse_output: state.parse_output.clone(), + diagnostics: Vec::new(), + lex_restart, + reparsed_bytes: 0, + reused_tokens: lex_restart.reused_tokens(), + token_count: state.tokens.len(), + } +} + fn apply_text_edits(state: &mut IncrementalState, edits_descending: &[Edit]) -> Result<()> { for edit in edits_descending { apply_text_edit_to_state(state, edit)?; @@ -112,6 +131,9 @@ fn full_reparse_after_edits( /// Apply edits incrementally. pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result { let total_changed = validate_edits(&state.source, edits)?; + if edits.is_empty() { + return Ok(unchanged_result(state)); + } let mut sorted_edits = edits.to_vec(); sorted_edits.sort_by_key(|edit| edit.start_byte); From 568ea27847a1022e7b62887264075e072a339fc2 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:21:51 -0400 Subject: [PATCH 32/48] test(parser): carry generation-neutral no-op proof into restart branch --- .../tests/incremental_parse_output.rs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/perl-parser/tests/incremental_parse_output.rs b/crates/perl-parser/tests/incremental_parse_output.rs index 980e48027d..6dcba3a737 100644 --- a/crates/perl-parser/tests/incremental_parse_output.rs +++ b/crates/perl-parser/tests/incremental_parse_output.rs @@ -68,6 +68,28 @@ fn initial_malformed_state_keeps_the_native_recovered_tree_and_diagnostics() -> Ok(()) } +#[test] +fn empty_edit_batch_preserves_the_current_generation_without_parser_work() -> TestResult { + let source = "my $x = ; print 1;"; + let mut state = IncrementalState::new(source.to_string()); + let before = state.parse_output.clone(); + let token_count = state.tokens.len(); + assert!(!before.diagnostics.is_empty(), "fixture must preserve recovered output"); + + let result = apply_edits(&mut state, &[])?; + + assert_eq!(state.source, source); + assert!(result.changed_ranges.is_empty()); + assert_eq!(result.reparsed_bytes, 0); + assert_eq!(result.reused_tokens, token_count); + assert_eq!(result.token_count, token_count); + assert_eq!(state.tokens.len(), token_count); + assert_output_equivalent(&state.parse_output, &before); + assert_output_equivalent(&result.parse_output, &before); + + Ok(()) +} + #[test] fn clean_to_malformed_edit_returns_the_current_native_parse_output() -> TestResult { let source = "my $x = 1; print 2;"; From c87b29cafbadce4e1856be3957779424a5acbed6 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:22:44 -0400 Subject: [PATCH 33/48] test(parser): expand restart receipts across stateful Perl edits --- .../tests/incremental_lexer_restart.rs | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/crates/perl-parser/tests/incremental_lexer_restart.rs b/crates/perl-parser/tests/incremental_lexer_restart.rs index 77e333a54d..2278371819 100644 --- a/crates/perl-parser/tests/incremental_lexer_restart.rs +++ b/crates/perl-parser/tests/incremental_lexer_restart.rs @@ -29,6 +29,39 @@ fn assert_tokens_equal(actual: &[Token], expected: &[Token]) { } } +fn replacing_edit(source: &str, needle: &str, replacement: &str) -> Result { + let start = source + .find(needle) + .ok_or_else(|| std::io::Error::other(format!("fixture needle {needle:?} is missing")))?; + Ok(Edit { + start_byte: start, + old_end_byte: start + needle.len(), + new_end_byte: start + replacement.len(), + new_text: replacement.to_string(), + }) +} + +#[test] +fn empty_edit_batch_reports_unchanged_without_lexer_or_parser_work() -> TestResult { + let source = "my $before = 1; my $after = 2;"; + let mut state = IncrementalState::new(source.to_string()); + let token_count = state.tokens.len(); + + let result = apply_edits(&mut state, &[])?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::Unchanged); + assert_eq!(result.lex_restart.restart_byte, source.len()); + assert_eq!(result.lex_restart.relexed_bytes, 0); + assert_eq!(result.lex_restart.reused_prefix_tokens, token_count); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_eq!(result.reused_tokens, token_count); + assert_eq!(result.reparsed_bytes, 0); + assert!(result.changed_ranges.is_empty()); + assert_eq!(state.source, source); + assert_tokens_equal(&state.tokens, &fresh_tokens(source)); + Ok(()) +} + #[test] fn late_equal_width_edit_retains_prefix_and_relexes_the_complete_suffix() -> TestResult { let source = "my $before = 1; my $target = 2; my $after = 3;"; @@ -55,6 +88,44 @@ fn late_equal_width_edit_retains_prefix_and_relexes_the_complete_suffix() -> Tes Ok(()) } +#[test] +fn stateful_and_source_boundary_edits_match_fresh_lexing() -> TestResult { + let fixtures = [ + ("division", "my $x = 10 / 2; my $after = 1;", "/ 2", "/ 3"), + ("regex", "my $ok = /foo/; my $after = 1;", "foo", "bar"), + ("quote-single", "my $x = q{foo}; my $after = 1;", "foo", "bar"), + ("quote-double", "my $x = qq{foo}; my $after = 1;", "foo", "bar"), + ("quote-words", "my @x = qw(foo bar); my $after = 1;", "foo", "baz"), + ("quote-command", "my $x = qx{echo foo}; my $after = 1;", "foo", "bar"), + ("substitution", "$x =~ s/foo/bar/; my $after = 1;", "foo", "baz"), + ("transliteration", "$x =~ tr/a-z/A-Z/; my $after = 1;", "a-z", "b-z"), + ("prototype", "sub f($$) { return 1; } my $after = 1;", "return 1", "return 2"), + ("unicode", "my $x = \"café\"; my $after = 1;", "é", "ø"), + ("crlf", "my $x = 1;\r\nmy $y = 2;\r\n", "= 2", "= 3"), + ( + "heredoc-body", + "my $value = < TestResult { let source = "my $value = $object->method(); my $after = 1;"; From d0e25aa83125191c90d3eea3b0983e19cbce8724 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:41:01 -0400 Subject: [PATCH 34/48] fix(parser): keep unit-test imports explicit --- crates/perl-parser/src/incremental/tests.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/perl-parser/src/incremental/tests.rs b/crates/perl-parser/src/incremental/tests.rs index 3b1223383c..07999958d9 100644 --- a/crates/perl-parser/src/incremental/tests.rs +++ b/crates/perl-parser/src/incremental/tests.rs @@ -1,5 +1,7 @@ use super::*; +use crate::Parser; use anyhow::Result; +use perl_ast::NodeKind; use proptest::prelude::*; #[derive(Clone, Debug)] From df5fae4cf0537dc75846fe0dabe685b5ccaee48f Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:52:29 -0400 Subject: [PATCH 35/48] fix(lexer): fail closed when checkpoint position identity shifts --- crates/perl-lexer/src/checkpoint/core.rs | 48 ++++++++++++++++-------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/crates/perl-lexer/src/checkpoint/core.rs b/crates/perl-lexer/src/checkpoint/core.rs index 2c86af4485..90aea337a3 100644 --- a/crates/perl-lexer/src/checkpoint/core.rs +++ b/crates/perl-lexer/src/checkpoint/core.rs @@ -132,10 +132,17 @@ impl LexerCheckpoint { } } - /// Create a default-state checkpoint at a specific position. + /// Create a default-state checkpoint at a specific byte position. + /// + /// Line and column remain the default summary values; the byte component is + /// aligned so the checkpoint remains structurally valid for compatibility + /// callers that only model a byte boundary. #[must_use] pub fn at_position(position: usize) -> Self { - Self { position, ..Self::new() } + let mut checkpoint = Self::new(); + checkpoint.position = position; + checkpoint.current_pos.byte = position; + checkpoint } /// Check whether this checkpoint is at the start of input. @@ -171,20 +178,22 @@ impl LexerCheckpoint { /// Apply an edit to source-relative checkpoint offsets. /// - /// Invalidated checkpoints retain the historical behavior of rewinding to - /// `start`. Call [`Self::try_apply_edit`] when the caller must distinguish a - /// transformed checkpoint from a conservative reset. + /// This compatibility method retains transformed offsets for inspection, + /// but any edit that cannot preserve complete replay state leaves an + /// explicitly unrestorable checkpoint. Call [`Self::try_apply_edit`] when + /// the caller must branch on that result. pub fn apply_edit(&mut self, start: usize, old_len: usize, new_len: usize) { let _ = self.try_apply_edit(start, old_len, new_len); } /// Apply an edit and report whether all required replay state survived. /// - /// An edit overlapping the replay position or another required state offset - /// invalidates the checkpoint and rewinds it to `start`, returning `false`. - /// Offsets after the replaced range are shifted. An edit beginning exactly - /// at an offset leaves it anchored before the replacement so the new text is - /// re-lexed. + /// An edit overlapping a required state offset invalidates the checkpoint. + /// A shift of the replay position also fails closed because byte counts do + /// not contain enough information to recompute line and column. Offsets are + /// still transformed for compatibility inspection, but [`Self::is_valid_for`] + /// rejects the result. An edit beginning exactly at an offset leaves it + /// anchored before the replacement so the new text is re-lexed. #[must_use] pub fn try_apply_edit(&mut self, start: usize, old_len: usize, new_len: usize) -> bool { let original_position = self.position; @@ -248,15 +257,17 @@ impl LexerCheckpoint { self.context = context; self.eof_emitted = false; if self.position != original_position { - self.current_pos = Position::start(); + self.mark_unrestorable(); + return false; } true } - /// Validate all source-relative checkpoint offsets for an input. + /// Validate all source-relative checkpoint offsets and replay identity for an input. #[must_use] pub fn is_valid_for(&self, input: &str) -> bool { - offset_is_valid(input, self.position) + self.current_pos.byte == self.position + && offset_is_valid(input, self.position) && offset_is_valid(input, self.line_start_offset) && self.line_start_offset <= self.position && self @@ -280,10 +291,17 @@ impl LexerCheckpoint { } fn invalidate_at(&mut self, start: usize) { - let mut reset = Self::new(); - reset.position = start; + let mut reset = Self::at_position(start); + reset.mark_unrestorable(); *self = reset; } + + fn mark_unrestorable(&mut self) { + self.current_pos = Position::start(); + if self.position == 0 { + self.current_pos.byte = usize::MAX; + } + } } fn transform_offset(offset: usize, start: usize, old_len: usize, new_len: usize) -> Option { From ec8d03a6a3e4353a8cd3b6c83ee4220adba1c677 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:53:12 -0400 Subject: [PATCH 36/48] test(lexer): prove shifted and invalidated checkpoints cannot restore --- .../tests/checkpoint_edit_validity.rs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 crates/perl-lexer/tests/checkpoint_edit_validity.rs diff --git a/crates/perl-lexer/tests/checkpoint_edit_validity.rs b/crates/perl-lexer/tests/checkpoint_edit_validity.rs new file mode 100644 index 0000000000..d46b3045ae --- /dev/null +++ b/crates/perl-lexer/tests/checkpoint_edit_validity.rs @@ -0,0 +1,47 @@ +use perl_lexer::checkpoint::{Checkpointable, LexerCheckpoint, QuoteOperatorCheckpoint}; +use perl_lexer::{PerlLexer, Position}; + +#[test] +fn shifted_replay_position_fails_closed_without_target_source_coordinates() { + let mut checkpoint = LexerCheckpoint::at_position(8); + checkpoint.current_pos = Position::new(8, 2, 3); + + let preserved = checkpoint.try_apply_edit(0, 0, 3); + + assert!(!preserved, "byte lengths cannot reconstruct shifted line and column state"); + assert_eq!(checkpoint.position, 11, "compatibility inspection keeps the shifted byte"); + assert_ne!(checkpoint.current_pos.byte, checkpoint.position); + + let lexer = PerlLexer::new("xxxmy $value = 1;\n"); + assert!(!checkpoint.is_valid_for("xxxmy $value = 1;\n")); + assert!(!lexer.can_restore(&checkpoint)); +} + +#[test] +fn compatibility_edit_cannot_turn_overlapped_quote_state_into_a_restorable_default() { + let mut checkpoint = LexerCheckpoint::at_position(12); + checkpoint.current_quote_op = Some(QuoteOperatorCheckpoint { + operator: "qq".to_string(), + delimiter: '{', + start_pos: 9, + }); + + checkpoint.apply_edit(8, 5, 1); + + assert_eq!(checkpoint.position, 8, "compatibility inspection retains the edit boundary"); + assert!(!checkpoint.is_valid_for("xxxxxxxxrest")); + + let lexer = PerlLexer::new("xxxxxxxxrest"); + assert!(!lexer.can_restore(&checkpoint)); +} + +#[test] +fn exact_boundary_edit_keeps_a_live_checkpoint_replayable() { + let mut checkpoint = LexerCheckpoint::at_position(8); + checkpoint.current_pos = Position::new(8, 1, 9); + + assert!(checkpoint.try_apply_edit(8, 2, 5)); + assert_eq!(checkpoint.position, 8); + assert_eq!(checkpoint.current_pos, Position::new(8, 1, 9)); + assert!(checkpoint.is_valid_for("12345678replacement")); +} From b8ecf5368d69353dffcfc5313cd59aa7e323ba98 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:54:19 -0400 Subject: [PATCH 37/48] refactor(parser): encapsulate committed incremental generations --- crates/perl-parser/src/incremental/state.rs | 91 ++++++++++++++++----- 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/crates/perl-parser/src/incremental/state.rs b/crates/perl-parser/src/incremental/state.rs index db92eb8b47..c454f53fdb 100644 --- a/crates/perl-parser/src/incremental/state.rs +++ b/crates/perl-parser/src/incremental/state.rs @@ -7,32 +7,32 @@ use perl_parser_core::error::ParseOutput; use perl_parser_core::parser::Parser; use ropey::Rope; +/// One internally consistent incremental parser generation. +/// +/// Generation-bearing fields are crate-private so external callers cannot +/// mutate source, tokens, checkpoints, AST, or parser output independently. +/// Use [`IncrementalState::new`], read-only accessors, and [`super::apply_edits`] +/// to move between committed generations. #[derive(Clone)] +#[non_exhaustive] pub struct IncrementalState { - pub rope: Rope, - pub line_index: LineIndex, - pub lex_checkpoints: Vec, - pub parse_checkpoints: Vec, + pub(super) rope: Rope, + pub(super) line_index: LineIndex, + pub(super) lex_checkpoints: Vec, + pub(super) parse_checkpoints: Vec, /// Authoritative native parser output for the current source. - /// - /// This is produced by `Parser::parse_with_recovery` and carries the AST, - /// ordered parser diagnostics, recovery count, budget usage, and early - /// termination state. Incremental consumers should use this field rather - /// than reconstructing parser state from an AST alone. - pub parse_output: ParseOutput, - /// Parsed AST compatibility field. - /// - /// This field mirrors [`Self::parse_output`]'s AST after every supported - /// state transition. It remains temporarily for compatibility with existing - /// callers and parse-checkpoint code. - #[deprecated(note = "Use parse_output.ast; this compatibility mirror will be removed.")] - pub ast: Node, - pub tokens: Vec, - pub source: String, + pub(super) parse_output: ParseOutput, + /// Parsed AST compatibility mirror. + #[deprecated(note = "Use parse_output(); this compatibility mirror will be removed.")] + pub(super) ast: Node, + pub(super) tokens: Vec, + pub(super) source: String, } impl IncrementalState { + /// Build the initial committed generation from source text. #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] + #[must_use] pub fn new(source: String) -> Self { let rope = Rope::from_str(&source); let line_index = LineIndex::new(&source); @@ -61,10 +61,63 @@ impl IncrementalState { } } + /// Current committed source text. + #[must_use] + pub fn source(&self) -> &str { + &self.source + } + + /// Rope view for the current committed source. + #[must_use] + pub fn rope(&self) -> &Rope { + &self.rope + } + + /// Line index for the current committed source. + #[must_use] + pub fn line_index(&self) -> &LineIndex { + &self.line_index + } + + /// Lexer restart summaries for the current committed token stream. + #[must_use] + pub fn lex_checkpoints(&self) -> &[LexCheckpoint] { + &self.lex_checkpoints + } + + /// Parser restart summaries for the current committed parse output. + #[must_use] + pub fn parse_checkpoints(&self) -> &[ParseCheckpoint] { + &self.parse_checkpoints + } + + /// Authoritative recovery-aware parser output for this generation. + #[must_use] + pub fn parse_output(&self) -> &ParseOutput { + &self.parse_output + } + + /// Compatibility AST view for the current generation. + #[deprecated(note = "Use parse_output().ast; this compatibility view will be removed.")] + #[must_use] + pub fn ast(&self) -> &Node { + &self.ast + } + + /// Current committed lexer token stream. + #[must_use] + pub fn tokens(&self) -> &[Token] { + &self.tokens + } + + /// Find the nearest lexer checkpoint at or before `byte`. + #[must_use] pub fn find_lex_checkpoint(&self, byte: usize) -> Option<&LexCheckpoint> { self.lex_checkpoints.iter().rev().find(|cp| cp.byte <= byte) } + /// Find the nearest parser checkpoint at or before `byte`. + #[must_use] pub fn find_parse_checkpoint(&self, byte: usize) -> Option<&ParseCheckpoint> { self.parse_checkpoints.iter().rev().find(|cp| cp.byte <= byte) } From a79c0b0f90cbd5fe03f997e3d633ea6f1b14aa97 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:56:45 -0400 Subject: [PATCH 38/48] test(parser): migrate parse-output contract to state accessors --- .../tests/incremental_parse_output.rs | 105 ++++++------------ 1 file changed, 34 insertions(+), 71 deletions(-) diff --git a/crates/perl-parser/tests/incremental_parse_output.rs b/crates/perl-parser/tests/incremental_parse_output.rs index 6dcba3a737..a692b677b9 100644 --- a/crates/perl-parser/tests/incremental_parse_output.rs +++ b/crates/perl-parser/tests/incremental_parse_output.rs @@ -12,139 +12,105 @@ fn fresh_output(source: &str) -> ParseOutput { } fn assert_output_equivalent(actual: &ParseOutput, expected: &ParseOutput) { - assert_eq!(actual.ast, expected.ast, "AST differs from a fresh recovered parse"); - assert_eq!( - actual.diagnostics, expected.diagnostics, - "ordered parser diagnostics differ from a fresh recovered parse" - ); + assert_eq!(actual.ast, expected.ast); + assert_eq!(actual.diagnostics, expected.diagnostics); assert_eq!(actual.terminated_early, expected.terminated_early); assert_eq!(actual.recovered_count, expected.recovered_count); - assert_eq!(actual.budget_usage.errors_emitted, expected.budget_usage.errors_emitted); - assert_eq!(actual.budget_usage.current_depth, expected.budget_usage.current_depth); - assert_eq!(actual.budget_usage.max_depth_reached, expected.budget_usage.max_depth_reached); - assert_eq!(actual.budget_usage.tokens_skipped, expected.budget_usage.tokens_skipped); - assert_eq!( - actual.budget_usage.recoveries_attempted, - expected.budget_usage.recoveries_attempted - ); + assert_eq!(actual.budget_usage, expected.budget_usage); } -fn apply_reference_edits(source: &str, edits: &[Edit]) -> Result { +fn apply_reference_edits(source: &str, edits: &[Edit]) -> String { let mut sorted = edits.to_vec(); sorted.sort_by_key(|edit| edit.start_byte); sorted.reverse(); - let mut result = source.to_string(); for edit in sorted { - if edit.start_byte > edit.old_end_byte || edit.old_end_byte > result.len() { - return Err("reference edit range is out of bounds"); - } - if !result.is_char_boundary(edit.start_byte) - || !result.is_char_boundary(edit.old_end_byte) - { - return Err("reference edit range is not on UTF-8 boundaries"); - } - if edit.new_end_byte != edit.start_byte + edit.new_text.len() { - return Err("reference edit new_end_byte is inconsistent"); - } result.replace_range(edit.start_byte..edit.old_end_byte, &edit.new_text); } - Ok(result) -} - -fn apply_reference_edit(source: &str, edit: &Edit) -> Result { - apply_reference_edits(source, std::slice::from_ref(edit)) + result } #[test] -fn initial_malformed_state_keeps_the_native_recovered_tree_and_diagnostics() -> TestResult { +fn initial_malformed_state_keeps_the_native_recovered_tree_and_diagnostics() { let source = "my $x = ; print 1;"; let state = IncrementalState::new(source.to_string()); let fresh = fresh_output(source); - - assert!(!fresh.diagnostics.is_empty(), "fixture must exercise structured recovery"); - assert_output_equivalent(&state.parse_output, &fresh); - - Ok(()) + assert!(!fresh.diagnostics.is_empty()); + assert_output_equivalent(state.parse_output(), &fresh); } #[test] -fn empty_edit_batch_preserves_the_current_generation_without_parser_work() -> TestResult { +fn empty_edit_batch_preserves_the_current_generation_without_work() -> TestResult { let source = "my $x = ; print 1;"; let mut state = IncrementalState::new(source.to_string()); - let before = state.parse_output.clone(); - let token_count = state.tokens.len(); - assert!(!before.diagnostics.is_empty(), "fixture must preserve recovered output"); + let before = state.parse_output().clone(); + let token_count = state.tokens().len(); let result = apply_edits(&mut state, &[])?; - assert_eq!(state.source, source); + assert_eq!(state.source(), source); assert!(result.changed_ranges.is_empty()); assert_eq!(result.reparsed_bytes, 0); assert_eq!(result.reused_tokens, token_count); assert_eq!(result.token_count, token_count); - assert_eq!(state.tokens.len(), token_count); - assert_output_equivalent(&state.parse_output, &before); + assert_eq!(state.tokens().len(), token_count); + assert_output_equivalent(state.parse_output(), &before); assert_output_equivalent(&result.parse_output, &before); - Ok(()) } #[test] fn clean_to_malformed_edit_returns_the_current_native_parse_output() -> TestResult { let source = "my $x = 1; print 2;"; - let start = source.find("= 1").ok_or("clean fixture lost its initializer")? + 2; + let start = source.find("= 1").ok_or("initializer missing")? + 2; let edit = Edit { start_byte: start, old_end_byte: start + 1, new_end_byte: start, new_text: String::new(), }; - let final_source = apply_reference_edit(source, &edit)?; + let final_source = apply_reference_edits(source, std::slice::from_ref(&edit)); let fresh = fresh_output(&final_source); - assert!(!fresh.diagnostics.is_empty(), "edited fixture must require recovery"); + assert!(!fresh.diagnostics.is_empty()); let mut state = IncrementalState::new(source.to_string()); let result = apply_edits(&mut state, &[edit])?; - assert_eq!(state.source, final_source); + assert_eq!(state.source(), final_source); assert_eq!(result.reparsed_bytes, final_source.len()); - assert_output_equivalent(&state.parse_output, &fresh); + assert_output_equivalent(state.parse_output(), &fresh); assert_output_equivalent(&result.parse_output, &fresh); - Ok(()) } #[test] fn malformed_to_clean_edit_removes_recovery_diagnostics_atomically() -> TestResult { let source = "my $x = ; print 2;"; - let start = source.find("= ;").ok_or("malformed fixture lost its insertion point")? + 2; + let start = source.find("= ;").ok_or("insertion point missing")? + 2; let edit = Edit { start_byte: start, old_end_byte: start, new_end_byte: start + 1, new_text: "1".to_string(), }; - let final_source = apply_reference_edit(source, &edit)?; + let final_source = apply_reference_edits(source, std::slice::from_ref(&edit)); let fresh = fresh_output(&final_source); - assert!(fresh.diagnostics.is_empty(), "repaired fixture should parse cleanly"); + assert!(fresh.diagnostics.is_empty()); let mut state = IncrementalState::new(source.to_string()); - assert!(!state.parse_output.diagnostics.is_empty()); + assert!(!state.parse_output().diagnostics.is_empty()); let result = apply_edits(&mut state, &[edit])?; - assert_eq!(state.source, final_source); - assert_eq!(result.reparsed_bytes, final_source.len()); - assert_output_equivalent(&state.parse_output, &fresh); + assert_eq!(state.source(), final_source); + assert_output_equivalent(state.parse_output(), &fresh); assert_output_equivalent(&result.parse_output, &fresh); - Ok(()) } #[test] fn oversized_batch_applies_every_edit_before_full_fallback() -> TestResult { let source = "my $left = 1;\nmy $right = 2;\n"; - let second_start = source.find("my $right").ok_or("second statement is missing")?; + let second_start = source.find("my $right").ok_or("second statement missing")?; let padding = " ".repeat(MAX_EDIT_SIZE / 2 + 1); let edits = vec![ Edit { @@ -160,25 +126,24 @@ fn oversized_batch_applies_every_edit_before_full_fallback() -> TestResult { new_text: padding, }, ]; - let final_source = apply_reference_edits(source, &edits)?; + let final_source = apply_reference_edits(source, &edits); let fresh = fresh_output(&final_source); let mut state = IncrementalState::new(source.to_string()); let result = apply_edits(&mut state, &edits)?; - assert_eq!(state.source, final_source); + assert_eq!(state.source(), final_source); assert_eq!(result.changed_ranges, vec![0..final_source.len()]); assert_eq!(result.reparsed_bytes, final_source.len()); - assert_output_equivalent(&state.parse_output, &fresh); + assert_output_equivalent(state.parse_output(), &fresh); assert_output_equivalent(&result.parse_output, &fresh); - Ok(()) } #[test] -fn invalid_overlapping_batch_leaves_the_previous_generation_untouched() -> TestResult { +fn invalid_overlapping_batch_leaves_the_previous_generation_untouched() { let source = "my $value = 12;"; - let literal = source.find("12").ok_or("literal is missing")?; + let literal = source.find("12").expect("literal missing"); let edits = [ Edit { start_byte: literal, @@ -196,11 +161,9 @@ fn invalid_overlapping_batch_leaves_the_previous_generation_untouched() -> TestR let before = fresh_output(source); let mut state = IncrementalState::new(source.to_string()); - let error = apply_edits(&mut state, &edits).expect_err("overlapping edits must be rejected"); + let error = apply_edits(&mut state, &edits).expect_err("overlap must fail"); assert!(error.to_string().contains("overlapping")); - assert_eq!(state.source, source); - assert_output_equivalent(&state.parse_output, &before); - - Ok(()) + assert_eq!(state.source(), source); + assert_output_equivalent(state.parse_output(), &before); } From 2a3500fb96845e54101d95a0194c07b43285bafa Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:57:55 -0400 Subject: [PATCH 39/48] refactor(parser): carry bounded generation state into restart branch --- crates/perl-parser/src/incremental/state.rs | 125 +++++++++++++------- 1 file changed, 83 insertions(+), 42 deletions(-) diff --git a/crates/perl-parser/src/incremental/state.rs b/crates/perl-parser/src/incremental/state.rs index 711ed88f23..ba2dbf3909 100644 --- a/crates/perl-parser/src/incremental/state.rs +++ b/crates/perl-parser/src/incremental/state.rs @@ -1,75 +1,116 @@ use crate::incremental::checkpoint::{LexCheckpoint, ParseCheckpoint, ScopeSnapshot}; -use crate::incremental::lex::lex_source_with_checkpoints; -use perl_lexer::Token; +use crate::incremental::lex::create_lex_checkpoints; +use perl_lexer::{PerlLexer, Token, TokenType}; use perl_line_index::LineIndex; use perl_parser_core::ast::{Node, NodeKind}; use perl_parser_core::error::ParseOutput; use perl_parser_core::parser::Parser; use ropey::Rope; +/// One internally consistent incremental parser generation. +/// +/// Generation-bearing fields are crate-private so external callers cannot +/// mutate source, tokens, checkpoints, AST, or parser output independently. +/// Use [`IncrementalState::new`], read-only accessors, and [`super::apply_edits`] +/// to move between committed generations. #[derive(Clone)] +#[non_exhaustive] pub struct IncrementalState { - pub rope: Rope, - pub line_index: LineIndex, - /// Compact compatibility summaries of live lexer restart boundaries. - /// - /// These summaries are captured from `PerlLexer::checkpoint()` while - /// lexing. The full live state is replayed and validated before any restart; - /// this summary alone never authorizes restoration. - pub lex_checkpoints: Vec, - pub parse_checkpoints: Vec, - /// Authoritative native parser output for the current source. - /// - /// This is produced by `Parser::parse_with_recovery` and carries the AST, - /// ordered parser diagnostics, recovery count, budget usage, and early - /// termination state. Incremental consumers should use this field rather - /// than reconstructing parser state from an AST alone. - pub parse_output: ParseOutput, - /// Parsed AST compatibility field. - /// - /// This field mirrors [`Self::parse_output`]'s AST after every supported - /// state transition. It remains temporarily for compatibility with existing - /// callers and parse-checkpoint code. - #[deprecated(note = "Use parse_output.ast; this compatibility mirror will be removed.")] - pub ast: Node, - pub tokens: Vec, - pub source: String, + pub(super) rope: Rope, + pub(super) line_index: LineIndex, + pub(super) lex_checkpoints: Vec, + pub(super) parse_checkpoints: Vec, + pub(super) parse_output: ParseOutput, + #[deprecated(note = "Use parse_output(); this compatibility mirror will be removed.")] + pub(super) ast: Node, + pub(super) tokens: Vec, + pub(super) source: String, } impl IncrementalState { + /// Build the initial committed generation from source text. #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] + #[must_use] pub fn new(source: String) -> Self { let rope = Rope::from_str(&source); let line_index = LineIndex::new(&source); let mut parser = Parser::new(&source); let parse_output = parser.parse_with_recovery(); let ast = parse_output.ast.clone(); - let lexed = lex_source_with_checkpoints(&source, &line_index); - let parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast); - Self { - rope, - line_index, - lex_checkpoints: lexed.checkpoints, - parse_checkpoints, - parse_output, - ast, - tokens: lexed.tokens, - source, + let mut lexer = PerlLexer::new(&source); + let mut tokens = Vec::new(); + while let Some(token) = lexer.next_token() { + if token.token_type == TokenType::EOF { + break; + } + tokens.push(token); } + let lex_checkpoints = create_lex_checkpoints(&tokens, &line_index); + let parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast); + Self { rope, line_index, lex_checkpoints, parse_checkpoints, parse_output, ast, tokens, source } + } + + /// Current committed source text. + #[must_use] + pub fn source(&self) -> &str { + &self.source + } + + /// Rope view for the current committed source. + #[must_use] + pub fn rope(&self) -> &Rope { + &self.rope + } + + /// Line index for the current committed source. + #[must_use] + pub fn line_index(&self) -> &LineIndex { + &self.line_index + } + + /// Lexer restart summaries for the current committed token stream. + #[must_use] + pub fn lex_checkpoints(&self) -> &[LexCheckpoint] { + &self.lex_checkpoints + } + + /// Parser restart summaries for the current committed parse output. + #[must_use] + pub fn parse_checkpoints(&self) -> &[ParseCheckpoint] { + &self.parse_checkpoints + } + + /// Authoritative recovery-aware parser output for this generation. + #[must_use] + pub fn parse_output(&self) -> &ParseOutput { + &self.parse_output + } + + /// Compatibility AST view for the current generation. + #[deprecated(note = "Use parse_output().ast; this compatibility view will be removed.")] + #[must_use] + pub fn ast(&self) -> &Node { + &self.ast + } + + /// Current committed lexer token stream. + #[must_use] + pub fn tokens(&self) -> &[Token] { + &self.tokens } + /// Find the nearest lexer checkpoint at or before `byte`. + #[must_use] pub fn find_lex_checkpoint(&self, byte: usize) -> Option<&LexCheckpoint> { self.lex_checkpoints.iter().rev().find(|cp| cp.byte <= byte) } + /// Find the nearest parser checkpoint at or before `byte`. + #[must_use] pub fn find_parse_checkpoint(&self, byte: usize) -> Option<&ParseCheckpoint> { self.parse_checkpoints.iter().rev().find(|cp| cp.byte <= byte) } - /// Refresh the authoritative parser output from the current source. - /// - /// The compatibility AST and parse checkpoints are updated from the same - /// recovered parse so the state cannot expose mixed parse generations. #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] pub(crate) fn refresh_parse_output(&mut self) { let mut parser = Parser::new(&self.source); From c4c8660ca9b08350fc54bb174214b5d2db05e647 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:59:36 -0400 Subject: [PATCH 40/48] test(parser): migrate restart contracts to state accessors --- .../tests/incremental_lexer_restart.rs | 18 +-- .../tests/incremental_parse_output.rs | 105 ++++++------------ 2 files changed, 43 insertions(+), 80 deletions(-) diff --git a/crates/perl-parser/tests/incremental_lexer_restart.rs b/crates/perl-parser/tests/incremental_lexer_restart.rs index 2278371819..1d0e2de711 100644 --- a/crates/perl-parser/tests/incremental_lexer_restart.rs +++ b/crates/perl-parser/tests/incremental_lexer_restart.rs @@ -45,7 +45,7 @@ fn replacing_edit(source: &str, needle: &str, replacement: &str) -> Result TestResult { let source = "my $before = 1; my $after = 2;"; let mut state = IncrementalState::new(source.to_string()); - let token_count = state.tokens.len(); + let token_count = state.tokens().len(); let result = apply_edits(&mut state, &[])?; @@ -57,8 +57,8 @@ fn empty_edit_batch_reports_unchanged_without_lexer_or_parser_work() -> TestResu assert_eq!(result.reused_tokens, token_count); assert_eq!(result.reparsed_bytes, 0); assert!(result.changed_ranges.is_empty()); - assert_eq!(state.source, source); - assert_tokens_equal(&state.tokens, &fresh_tokens(source)); + assert_eq!(state.source(), source); + assert_tokens_equal(state.tokens(), &fresh_tokens(source)); Ok(()) } @@ -81,10 +81,10 @@ fn late_equal_width_edit_retains_prefix_and_relexes_the_complete_suffix() -> Tes assert_eq!(result.lex_restart.reused_suffix_tokens, 0); assert_eq!( result.lex_restart.relexed_bytes, - state.source.len() - result.lex_restart.restart_byte + state.source().len() - result.lex_restart.restart_byte ); assert_eq!(result.reused_tokens, result.lex_restart.reused_tokens()); - assert_tokens_equal(&state.tokens, &fresh_tokens(&state.source)); + assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); Ok(()) } @@ -121,7 +121,7 @@ fn stateful_and_source_boundary_edits_match_fresh_lexing() -> TestResult { "{name} unexpectedly abandoned the live checkpoint path" ); assert_eq!(result.lex_restart.reused_suffix_tokens, 0, "{name}"); - assert_tokens_equal(&state.tokens, &fresh_tokens(&state.source)); + assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); } Ok(()) } @@ -141,7 +141,7 @@ fn method_context_edit_matches_fresh_lexing_after_complete_state_restore() -> Te assert_eq!(result.lex_restart.strategy, LexRestartStrategy::LiveCheckpointToEof); assert_eq!(result.lex_restart.reused_suffix_tokens, 0); - assert_tokens_equal(&state.tokens, &fresh_tokens(&state.source)); + assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); Ok(()) } @@ -162,7 +162,7 @@ fn large_edit_reports_full_relex_instead_of_checkpoint_reuse() -> TestResult { assert_eq!(result.lex_restart.restart_byte, 0); assert_eq!(result.lex_restart.reused_prefix_tokens, 0); assert_eq!(result.lex_restart.reused_suffix_tokens, 0); - assert_eq!(result.lex_restart.relexed_bytes, state.source.len()); - assert_tokens_equal(&state.tokens, &fresh_tokens(&state.source)); + assert_eq!(result.lex_restart.relexed_bytes, state.source().len()); + assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); Ok(()) } diff --git a/crates/perl-parser/tests/incremental_parse_output.rs b/crates/perl-parser/tests/incremental_parse_output.rs index 6dcba3a737..a692b677b9 100644 --- a/crates/perl-parser/tests/incremental_parse_output.rs +++ b/crates/perl-parser/tests/incremental_parse_output.rs @@ -12,139 +12,105 @@ fn fresh_output(source: &str) -> ParseOutput { } fn assert_output_equivalent(actual: &ParseOutput, expected: &ParseOutput) { - assert_eq!(actual.ast, expected.ast, "AST differs from a fresh recovered parse"); - assert_eq!( - actual.diagnostics, expected.diagnostics, - "ordered parser diagnostics differ from a fresh recovered parse" - ); + assert_eq!(actual.ast, expected.ast); + assert_eq!(actual.diagnostics, expected.diagnostics); assert_eq!(actual.terminated_early, expected.terminated_early); assert_eq!(actual.recovered_count, expected.recovered_count); - assert_eq!(actual.budget_usage.errors_emitted, expected.budget_usage.errors_emitted); - assert_eq!(actual.budget_usage.current_depth, expected.budget_usage.current_depth); - assert_eq!(actual.budget_usage.max_depth_reached, expected.budget_usage.max_depth_reached); - assert_eq!(actual.budget_usage.tokens_skipped, expected.budget_usage.tokens_skipped); - assert_eq!( - actual.budget_usage.recoveries_attempted, - expected.budget_usage.recoveries_attempted - ); + assert_eq!(actual.budget_usage, expected.budget_usage); } -fn apply_reference_edits(source: &str, edits: &[Edit]) -> Result { +fn apply_reference_edits(source: &str, edits: &[Edit]) -> String { let mut sorted = edits.to_vec(); sorted.sort_by_key(|edit| edit.start_byte); sorted.reverse(); - let mut result = source.to_string(); for edit in sorted { - if edit.start_byte > edit.old_end_byte || edit.old_end_byte > result.len() { - return Err("reference edit range is out of bounds"); - } - if !result.is_char_boundary(edit.start_byte) - || !result.is_char_boundary(edit.old_end_byte) - { - return Err("reference edit range is not on UTF-8 boundaries"); - } - if edit.new_end_byte != edit.start_byte + edit.new_text.len() { - return Err("reference edit new_end_byte is inconsistent"); - } result.replace_range(edit.start_byte..edit.old_end_byte, &edit.new_text); } - Ok(result) -} - -fn apply_reference_edit(source: &str, edit: &Edit) -> Result { - apply_reference_edits(source, std::slice::from_ref(edit)) + result } #[test] -fn initial_malformed_state_keeps_the_native_recovered_tree_and_diagnostics() -> TestResult { +fn initial_malformed_state_keeps_the_native_recovered_tree_and_diagnostics() { let source = "my $x = ; print 1;"; let state = IncrementalState::new(source.to_string()); let fresh = fresh_output(source); - - assert!(!fresh.diagnostics.is_empty(), "fixture must exercise structured recovery"); - assert_output_equivalent(&state.parse_output, &fresh); - - Ok(()) + assert!(!fresh.diagnostics.is_empty()); + assert_output_equivalent(state.parse_output(), &fresh); } #[test] -fn empty_edit_batch_preserves_the_current_generation_without_parser_work() -> TestResult { +fn empty_edit_batch_preserves_the_current_generation_without_work() -> TestResult { let source = "my $x = ; print 1;"; let mut state = IncrementalState::new(source.to_string()); - let before = state.parse_output.clone(); - let token_count = state.tokens.len(); - assert!(!before.diagnostics.is_empty(), "fixture must preserve recovered output"); + let before = state.parse_output().clone(); + let token_count = state.tokens().len(); let result = apply_edits(&mut state, &[])?; - assert_eq!(state.source, source); + assert_eq!(state.source(), source); assert!(result.changed_ranges.is_empty()); assert_eq!(result.reparsed_bytes, 0); assert_eq!(result.reused_tokens, token_count); assert_eq!(result.token_count, token_count); - assert_eq!(state.tokens.len(), token_count); - assert_output_equivalent(&state.parse_output, &before); + assert_eq!(state.tokens().len(), token_count); + assert_output_equivalent(state.parse_output(), &before); assert_output_equivalent(&result.parse_output, &before); - Ok(()) } #[test] fn clean_to_malformed_edit_returns_the_current_native_parse_output() -> TestResult { let source = "my $x = 1; print 2;"; - let start = source.find("= 1").ok_or("clean fixture lost its initializer")? + 2; + let start = source.find("= 1").ok_or("initializer missing")? + 2; let edit = Edit { start_byte: start, old_end_byte: start + 1, new_end_byte: start, new_text: String::new(), }; - let final_source = apply_reference_edit(source, &edit)?; + let final_source = apply_reference_edits(source, std::slice::from_ref(&edit)); let fresh = fresh_output(&final_source); - assert!(!fresh.diagnostics.is_empty(), "edited fixture must require recovery"); + assert!(!fresh.diagnostics.is_empty()); let mut state = IncrementalState::new(source.to_string()); let result = apply_edits(&mut state, &[edit])?; - assert_eq!(state.source, final_source); + assert_eq!(state.source(), final_source); assert_eq!(result.reparsed_bytes, final_source.len()); - assert_output_equivalent(&state.parse_output, &fresh); + assert_output_equivalent(state.parse_output(), &fresh); assert_output_equivalent(&result.parse_output, &fresh); - Ok(()) } #[test] fn malformed_to_clean_edit_removes_recovery_diagnostics_atomically() -> TestResult { let source = "my $x = ; print 2;"; - let start = source.find("= ;").ok_or("malformed fixture lost its insertion point")? + 2; + let start = source.find("= ;").ok_or("insertion point missing")? + 2; let edit = Edit { start_byte: start, old_end_byte: start, new_end_byte: start + 1, new_text: "1".to_string(), }; - let final_source = apply_reference_edit(source, &edit)?; + let final_source = apply_reference_edits(source, std::slice::from_ref(&edit)); let fresh = fresh_output(&final_source); - assert!(fresh.diagnostics.is_empty(), "repaired fixture should parse cleanly"); + assert!(fresh.diagnostics.is_empty()); let mut state = IncrementalState::new(source.to_string()); - assert!(!state.parse_output.diagnostics.is_empty()); + assert!(!state.parse_output().diagnostics.is_empty()); let result = apply_edits(&mut state, &[edit])?; - assert_eq!(state.source, final_source); - assert_eq!(result.reparsed_bytes, final_source.len()); - assert_output_equivalent(&state.parse_output, &fresh); + assert_eq!(state.source(), final_source); + assert_output_equivalent(state.parse_output(), &fresh); assert_output_equivalent(&result.parse_output, &fresh); - Ok(()) } #[test] fn oversized_batch_applies_every_edit_before_full_fallback() -> TestResult { let source = "my $left = 1;\nmy $right = 2;\n"; - let second_start = source.find("my $right").ok_or("second statement is missing")?; + let second_start = source.find("my $right").ok_or("second statement missing")?; let padding = " ".repeat(MAX_EDIT_SIZE / 2 + 1); let edits = vec![ Edit { @@ -160,25 +126,24 @@ fn oversized_batch_applies_every_edit_before_full_fallback() -> TestResult { new_text: padding, }, ]; - let final_source = apply_reference_edits(source, &edits)?; + let final_source = apply_reference_edits(source, &edits); let fresh = fresh_output(&final_source); let mut state = IncrementalState::new(source.to_string()); let result = apply_edits(&mut state, &edits)?; - assert_eq!(state.source, final_source); + assert_eq!(state.source(), final_source); assert_eq!(result.changed_ranges, vec![0..final_source.len()]); assert_eq!(result.reparsed_bytes, final_source.len()); - assert_output_equivalent(&state.parse_output, &fresh); + assert_output_equivalent(state.parse_output(), &fresh); assert_output_equivalent(&result.parse_output, &fresh); - Ok(()) } #[test] -fn invalid_overlapping_batch_leaves_the_previous_generation_untouched() -> TestResult { +fn invalid_overlapping_batch_leaves_the_previous_generation_untouched() { let source = "my $value = 12;"; - let literal = source.find("12").ok_or("literal is missing")?; + let literal = source.find("12").expect("literal missing"); let edits = [ Edit { start_byte: literal, @@ -196,11 +161,9 @@ fn invalid_overlapping_batch_leaves_the_previous_generation_untouched() -> TestR let before = fresh_output(source); let mut state = IncrementalState::new(source.to_string()); - let error = apply_edits(&mut state, &edits).expect_err("overlapping edits must be rejected"); + let error = apply_edits(&mut state, &edits).expect_err("overlap must fail"); assert!(error.to_string().contains("overlapping")); - assert_eq!(state.source, source); - assert_output_equivalent(&state.parse_output, &before); - - Ok(()) + assert_eq!(state.source(), source); + assert_output_equivalent(state.parse_output(), &before); } From a25f2d65a6cbf1999b8195e0dac5e266877ec7a7 Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:20:23 -0400 Subject: [PATCH 41/48] fix(parser): preserve read-only source compatibility --- crates/perl-parser/src/incremental/state.rs | 58 +++++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/crates/perl-parser/src/incremental/state.rs b/crates/perl-parser/src/incremental/state.rs index c454f53fdb..a1828fab0f 100644 --- a/crates/perl-parser/src/incremental/state.rs +++ b/crates/perl-parser/src/incremental/state.rs @@ -6,6 +6,18 @@ use perl_parser_core::ast::{Node, NodeKind}; use perl_parser_core::error::ParseOutput; use perl_parser_core::parser::Parser; use ropey::Rope; +use std::ops::Deref; + +/// Read-only compatibility view for legacy field-style source access. +/// +/// `IncrementalState` intentionally implements `Deref` but not `DerefMut` for +/// this view. Existing consumers may continue to read `state.source`, while +/// source replacement remains private to the committed-generation machinery. +#[doc(hidden)] +#[derive(Clone)] +pub struct IncrementalStateReadView { + pub source: String, +} /// One internally consistent incremental parser generation. /// @@ -13,6 +25,25 @@ use ropey::Rope; /// mutate source, tokens, checkpoints, AST, or parser output independently. /// Use [`IncrementalState::new`], read-only accessors, and [`super::apply_edits`] /// to move between committed generations. +/// +/// Legacy field-style source reads remain available through an immutable +/// compatibility view: +/// +/// ``` +/// use perl_parser::incremental::IncrementalState; +/// +/// let state = IncrementalState::new("my $x = 1;".to_string()); +/// assert_eq!(state.source.len(), state.source().len()); +/// ``` +/// +/// The view does not grant mutation authority: +/// +/// ```compile_fail +/// use perl_parser::incremental::IncrementalState; +/// +/// let mut state = IncrementalState::new("my $x = 1;".to_string()); +/// state.source.push_str("\n"); +/// ``` #[derive(Clone)] #[non_exhaustive] pub struct IncrementalState { @@ -26,7 +57,15 @@ pub struct IncrementalState { #[deprecated(note = "Use parse_output(); this compatibility mirror will be removed.")] pub(super) ast: Node, pub(super) tokens: Vec, - pub(super) source: String, + pub(super) read_view: IncrementalStateReadView, +} + +impl Deref for IncrementalState { + type Target = IncrementalStateReadView; + + fn deref(&self) -> &Self::Target { + &self.read_view + } } impl IncrementalState { @@ -57,14 +96,14 @@ impl IncrementalState { parse_output, ast, tokens, - source, + read_view: IncrementalStateReadView { source }, } } /// Current committed source text. #[must_use] pub fn source(&self) -> &str { - &self.source + &self.read_view.source } /// Rope view for the current committed source. @@ -122,13 +161,24 @@ impl IncrementalState { self.parse_checkpoints.iter().rev().find(|cp| cp.byte <= byte) } + /// Replace the text-bearing portion of a staged generation. + /// + /// This remains crate-private so source, rope, and line-index identity cannot + /// be changed independently by consumers. Callers finish rebuilding tokens, + /// checkpoints, and parser output before publishing the staged state. + pub(super) fn replace_source_text(&mut self, source: String) { + self.rope = Rope::from_str(&source); + self.line_index = LineIndex::new(&source); + self.read_view.source = source; + } + /// Refresh the authoritative parser output from the current source. /// /// The compatibility AST and parse checkpoints are updated from the same /// recovered parse so the state cannot expose mixed parse generations. #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] pub(crate) fn refresh_parse_output(&mut self) { - let mut parser = Parser::new(&self.source); + let mut parser = Parser::new(self.source()); let parse_output = parser.parse_with_recovery(); self.parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast); self.ast = parse_output.ast.clone(); From 658460a892cd9c800307ce3013c09da817bc9cec Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:21:08 -0400 Subject: [PATCH 42/48] refactor(parser): stage source replacement behind state API --- crates/perl-parser/src/incremental/reparse.rs | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/crates/perl-parser/src/incremental/reparse.rs b/crates/perl-parser/src/incremental/reparse.rs index 6ec03b595a..f906342c25 100644 --- a/crates/perl-parser/src/incremental/reparse.rs +++ b/crates/perl-parser/src/incremental/reparse.rs @@ -23,20 +23,18 @@ fn shift_offset(offset: usize, byte_shift: isize) -> usize { } pub(crate) fn apply_text_edit_to_state(state: &mut IncrementalState, edit: &Edit) -> Result<()> { - let old_end = edit.old_end_byte.min(state.source.len()); - let start = edit.start_byte.min(state.source.len()); - if !state.source.is_char_boundary(start) || !state.source.is_char_boundary(old_end) { + let old_end = edit.old_end_byte.min(state.source().len()); + let start = edit.start_byte.min(state.source().len()); + if !state.source().is_char_boundary(start) || !state.source().is_char_boundary(old_end) { anyhow::bail!("edit range is not on UTF-8 boundaries"); } let mut new_source = - String::with_capacity(state.source.len() - (old_end - start) + edit.new_text.len()); - new_source.push_str(&state.source[..start]); + String::with_capacity(state.source().len() - (old_end - start) + edit.new_text.len()); + new_source.push_str(&state.source()[..start]); new_source.push_str(&edit.new_text); - new_source.push_str(&state.source[old_end..]); - state.source = new_source; - state.rope = Rope::from_str(&state.source); - state.line_index = perl_line_index::LineIndex::new(&state.source); + new_source.push_str(&state.source()[old_end..]); + state.replace_source_text(new_source); Ok(()) } @@ -49,13 +47,13 @@ pub(crate) fn apply_single_edit( apply_text_edit_to_state(state, edit)?; anyhow::bail!("No checkpoint found"); }; - let old_end = edit.old_end_byte.min(state.source.len()); - let start = edit.start_byte.min(state.source.len()); + let old_end = edit.old_end_byte.min(state.source().len()); + let start = edit.start_byte.min(state.source().len()); let byte_shift = edit.new_text.len() as isize - (old_end - start) as isize; apply_text_edit_to_state(state, edit)?; use perl_lexer::{Checkpointable, LexerCheckpoint, Position}; - let mut lexer = PerlLexer::new(&state.source); + let mut lexer = PerlLexer::new(state.source()); let mut lex_cp = LexerCheckpoint::new(); lex_cp.position = cp.byte; lex_cp.mode = cp.mode; @@ -119,7 +117,8 @@ pub(crate) fn apply_single_edit( pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result { state.refresh_parse_output(); - let mut lexer = PerlLexer::new(&state.source); + let source = state.source().to_owned(); + let mut lexer = PerlLexer::new(&source); let mut tokens = Vec::new(); while let Some(token) = lexer.next_token() { if token.token_type == TokenType::EOF { @@ -128,14 +127,14 @@ pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result Date: Tue, 11 Aug 2026 18:44:37 -0400 Subject: [PATCH 43/48] fix(parser): keep lexer checkpoints read-compatible --- crates/perl-parser/src/incremental/reparse.rs | 8 ++--- crates/perl-parser/src/incremental/state.rs | 33 +++++++++++++------ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/crates/perl-parser/src/incremental/reparse.rs b/crates/perl-parser/src/incremental/reparse.rs index f906342c25..c01c1f7e89 100644 --- a/crates/perl-parser/src/incremental/reparse.rs +++ b/crates/perl-parser/src/incremental/reparse.rs @@ -1,6 +1,4 @@ -use crate::incremental::{ - IncrementalState, diagnostics::ReparseResult, edit::Edit, lex::create_lex_checkpoints, -}; +use crate::incremental::{IncrementalState, diagnostics::ReparseResult, edit::Edit}; use anyhow::Result; use perl_lexer::{PerlLexer, TokenType}; use ropey::Rope; @@ -111,7 +109,7 @@ pub(crate) fn apply_single_edit( } } state.tokens.splice(start_idx.., new_tokens); - state.lex_checkpoints = create_lex_checkpoints(&state.tokens, &state.line_index); + state.refresh_lex_checkpoints(); Ok(SingleEditReparse { range: cp.byte..last, reused_tokens, token_count: state.tokens.len() }) } @@ -129,7 +127,7 @@ pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result, } /// One internally consistent incremental parser generation. @@ -26,14 +28,15 @@ pub struct IncrementalStateReadView { /// Use [`IncrementalState::new`], read-only accessors, and [`super::apply_edits`] /// to move between committed generations. /// -/// Legacy field-style source reads remain available through an immutable -/// compatibility view: +/// Legacy field-style reads remain available through an immutable compatibility +/// view: /// /// ``` /// use perl_parser::incremental::IncrementalState; /// /// let state = IncrementalState::new("my $x = 1;".to_string()); /// assert_eq!(state.source.len(), state.source().len()); +/// assert_eq!(state.lex_checkpoints.len(), state.lex_checkpoints().len()); /// ``` /// /// The view does not grant mutation authority: @@ -44,12 +47,18 @@ pub struct IncrementalStateReadView { /// let mut state = IncrementalState::new("my $x = 1;".to_string()); /// state.source.push_str("\n"); /// ``` +/// +/// ```compile_fail +/// use perl_parser::incremental::IncrementalState; +/// +/// let mut state = IncrementalState::new("my $x = 1;".to_string()); +/// state.lex_checkpoints.clear(); +/// ``` #[derive(Clone)] #[non_exhaustive] pub struct IncrementalState { pub(super) rope: Rope, pub(super) line_index: LineIndex, - pub(super) lex_checkpoints: Vec, pub(super) parse_checkpoints: Vec, /// Authoritative native parser output for the current source. pub(super) parse_output: ParseOutput, @@ -91,12 +100,11 @@ impl IncrementalState { Self { rope, line_index, - lex_checkpoints, parse_checkpoints, parse_output, ast, tokens, - read_view: IncrementalStateReadView { source }, + read_view: IncrementalStateReadView { source, lex_checkpoints }, } } @@ -121,7 +129,7 @@ impl IncrementalState { /// Lexer restart summaries for the current committed token stream. #[must_use] pub fn lex_checkpoints(&self) -> &[LexCheckpoint] { - &self.lex_checkpoints + &self.read_view.lex_checkpoints } /// Parser restart summaries for the current committed parse output. @@ -152,7 +160,7 @@ impl IncrementalState { /// Find the nearest lexer checkpoint at or before `byte`. #[must_use] pub fn find_lex_checkpoint(&self, byte: usize) -> Option<&LexCheckpoint> { - self.lex_checkpoints.iter().rev().find(|cp| cp.byte <= byte) + self.read_view.lex_checkpoints.iter().rev().find(|cp| cp.byte <= byte) } /// Find the nearest parser checkpoint at or before `byte`. @@ -172,6 +180,11 @@ impl IncrementalState { self.read_view.source = source; } + /// Rebuild lexer checkpoints from the staged token stream and line index. + pub(super) fn refresh_lex_checkpoints(&mut self) { + self.read_view.lex_checkpoints = create_lex_checkpoints(&self.tokens, &self.line_index); + } + /// Refresh the authoritative parser output from the current source. /// /// The compatibility AST and parse checkpoints are updated from the same From 5fce131c5820522f461a351f95dd90931ceedf49 Mon Sep 17 00:00:00 2001 From: Steven Zimmerman Date: Tue, 11 Aug 2026 22:51:51 -0400 Subject: [PATCH 44/48] fix(incremental): keep live checkpoints restorable --- crates/perl-lexer/src/checkpoint/core.rs | 17 +++++++++-------- crates/perl-parser/src/incremental/tests.rs | 2 ++ .../tests/incremental_parse_output.rs | 11 +++++++++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/crates/perl-lexer/src/checkpoint/core.rs b/crates/perl-lexer/src/checkpoint/core.rs index 90aea337a3..e15a208b97 100644 --- a/crates/perl-lexer/src/checkpoint/core.rs +++ b/crates/perl-lexer/src/checkpoint/core.rs @@ -234,13 +234,13 @@ impl LexerCheckpoint { .map(|shifted| *start_position = shifted) .is_some() } - CheckpointContext::Regex { flags_position, .. } => flags_position.as_mut().is_none_or( - |flags| { + CheckpointContext::Regex { flags_position, .. } => { + flags_position.as_mut().is_none_or(|flags| { transform_offset(*flags, start, old_len, new_len) .map(|shifted| *flags = shifted) .is_some() - }, - ), + }) + } CheckpointContext::Normal | CheckpointContext::Heredoc { .. } | CheckpointContext::QuoteLike { .. } => true, @@ -266,7 +266,8 @@ impl LexerCheckpoint { /// Validate all source-relative checkpoint offsets and replay identity for an input. #[must_use] pub fn is_valid_for(&self, input: &str) -> bool { - self.current_pos.byte == self.position + self.current_pos.byte != usize::MAX + && offset_is_valid(input, self.current_pos.byte) && offset_is_valid(input, self.position) && offset_is_valid(input, self.line_start_offset) && self.line_start_offset <= self.position @@ -298,9 +299,9 @@ impl LexerCheckpoint { fn mark_unrestorable(&mut self) { self.current_pos = Position::start(); - if self.position == 0 { - self.current_pos.byte = usize::MAX; - } + // Keep the invalidation marker distinct from a live checkpoint whose + // legacy line/column summary has not advanced with ordinary lexing. + self.current_pos.byte = usize::MAX; } } diff --git a/crates/perl-parser/src/incremental/tests.rs b/crates/perl-parser/src/incremental/tests.rs index 3b1223383c..12fd851274 100644 --- a/crates/perl-parser/src/incremental/tests.rs +++ b/crates/perl-parser/src/incremental/tests.rs @@ -1,5 +1,7 @@ use super::*; use anyhow::Result; +use perl_parser_core::ast::NodeKind; +use perl_parser_core::parser::Parser; use proptest::prelude::*; #[derive(Clone, Debug)] diff --git a/crates/perl-parser/tests/incremental_parse_output.rs b/crates/perl-parser/tests/incremental_parse_output.rs index a692b677b9..dd495ead0f 100644 --- a/crates/perl-parser/tests/incremental_parse_output.rs +++ b/crates/perl-parser/tests/incremental_parse_output.rs @@ -2,7 +2,7 @@ //! Differential tests for the incremental native parse-output contract. use perl_parser::incremental::MAX_EDIT_SIZE; -use perl_parser::{Edit, IncrementalState, ParseOutput, Parser, apply_edits}; +use perl_parser::{apply_edits, Edit, IncrementalState, ParseOutput, Parser}; type TestResult = Result<(), Box>; @@ -16,7 +16,14 @@ fn assert_output_equivalent(actual: &ParseOutput, expected: &ParseOutput) { assert_eq!(actual.diagnostics, expected.diagnostics); assert_eq!(actual.terminated_early, expected.terminated_early); assert_eq!(actual.recovered_count, expected.recovered_count); - assert_eq!(actual.budget_usage, expected.budget_usage); + assert_eq!(actual.budget_usage.errors_emitted, expected.budget_usage.errors_emitted); + assert_eq!(actual.budget_usage.current_depth, expected.budget_usage.current_depth); + assert_eq!(actual.budget_usage.max_depth_reached, expected.budget_usage.max_depth_reached); + assert_eq!(actual.budget_usage.tokens_skipped, expected.budget_usage.tokens_skipped); + assert_eq!( + actual.budget_usage.recoveries_attempted, + expected.budget_usage.recoveries_attempted + ); } fn apply_reference_edits(source: &str, edits: &[Edit]) -> String { From ce179ede4623d970ef4fdabab7db7bc667c1cd24 Mon Sep 17 00:00:00 2001 From: Steven Zimmerman Date: Wed, 12 Aug 2026 00:56:55 -0400 Subject: [PATCH 45/48] fix(parser): fail closed on timeout-sensitive checkpoints --- crates/perl-lexer/src/checkpoint/core.rs | 20 ++++++++-- crates/perl-lexer/src/checkpoint_impl.rs | 39 +++++++------------ crates/perl-parser/src/incremental/lex.rs | 19 +++++---- .../tests/incremental_lexer_restart.rs | 38 +++++++++++++++--- 4 files changed, 74 insertions(+), 42 deletions(-) diff --git a/crates/perl-lexer/src/checkpoint/core.rs b/crates/perl-lexer/src/checkpoint/core.rs index e15a208b97..eed5c2bbf4 100644 --- a/crates/perl-lexer/src/checkpoint/core.rs +++ b/crates/perl-lexer/src/checkpoint/core.rs @@ -25,11 +25,13 @@ pub struct QuoteOperatorCheckpoint { /// A checkpoint that captures all mutable lexer state needed for token replay. /// -/// Input references and the wall-clock timeout origin are deliberately not -/// persisted. Restore targets supply the edited input, retain their configured -/// lexer policy, and begin a fresh operation-local timeout budget. +/// Input references are deliberately not persisted. The monotonic timeout +/// origin is retained so restoring a checkpoint cannot silently grant a fresh +/// heredoc timeout budget. #[derive(Debug, Clone, PartialEq)] pub struct LexerCheckpoint { + /// Monotonic origin used by timeout-sensitive lexer paths. + pub start_time: std::time::Instant, /// Current position in the input. pub position: usize, /// Current lexer mode (`ExpectTerm`, `ExpectOperator`, etc.). @@ -110,6 +112,7 @@ impl LexerCheckpoint { #[must_use] pub fn new() -> Self { Self { + start_time: std::time::Instant::now(), position: 0, mode: LexerMode::ExpectTerm, delimiter_stack: Vec::new(), @@ -151,6 +154,14 @@ impl LexerCheckpoint { self.position == 0 } + /// Whether restoring this checkpoint would re-enter a wall-clock-bounded + /// lexer path. Callers must fall back to a full re-lex when this is true + /// unless they can prove the timeout origin is safe for the operation. + #[must_use] + pub fn is_timeout_sensitive(&self) -> bool { + !self.pending_heredocs.is_empty() + } + /// Calculate the difference between two checkpoints. #[must_use] pub fn diff(&self, other: &Self) -> super::CheckpointDiff { @@ -348,7 +359,8 @@ pub trait Checkpointable { /// Restore mutable replay state into a lexer for the target input. /// - /// The target lexer retains its configured policy and fresh timeout origin. + /// The target lexer retains its configured policy and checkpoint timeout + /// origin. fn restore(&mut self, checkpoint: &LexerCheckpoint); /// Check whether every source-relative checkpoint offset is valid. diff --git a/crates/perl-lexer/src/checkpoint_impl.rs b/crates/perl-lexer/src/checkpoint_impl.rs index ba436a7ee4..9e3a87e1ab 100644 --- a/crates/perl-lexer/src/checkpoint_impl.rs +++ b/crates/perl-lexer/src/checkpoint_impl.rs @@ -1,6 +1,4 @@ -use crate::checkpoint::{ - Checkpointable, PendingHeredocCheckpoint, QuoteOperatorCheckpoint, -}; +use crate::checkpoint::{Checkpointable, PendingHeredocCheckpoint, QuoteOperatorCheckpoint}; use crate::heredoc::HeredocSpec; use crate::quote_handler::QuoteOperatorInfo; use crate::{LexerCheckpoint, LexerMode, PerlLexer, checkpoint}; @@ -30,6 +28,7 @@ impl Checkpointable for PerlLexer<'_> { }; LexerCheckpoint { + start_time: self.start_time, position: self.position, mode: self.mode, delimiter_stack: self.delimiter_stack.clone(), @@ -53,12 +52,10 @@ impl Checkpointable for PerlLexer<'_> { .collect(), line_start_offset: self.line_start_offset, emit_heredoc_body_tokens: self.emit_heredoc_body_tokens, - current_quote_op: self.current_quote_op.as_ref().map(|quote| { - QuoteOperatorCheckpoint { - operator: quote.operator.clone(), - delimiter: quote.delimiter, - start_pos: quote.start_pos, - } + current_quote_op: self.current_quote_op.as_ref().map(|quote| QuoteOperatorCheckpoint { + operator: quote.operator.clone(), + delimiter: quote.delimiter, + start_pos: quote.start_pos, }), qw_recovery_enabled: self.qw_recovery_enabled, eof_emitted: self.eof_emitted, @@ -67,6 +64,7 @@ impl Checkpointable for PerlLexer<'_> { } fn restore(&mut self, checkpoint: &LexerCheckpoint) { + self.start_time = checkpoint.start_time; self.position = checkpoint.position; self.mode = checkpoint.mode; self.delimiter_stack.clone_from(&checkpoint.delimiter_stack); @@ -90,13 +88,12 @@ impl Checkpointable for PerlLexer<'_> { .collect(); self.line_start_offset = checkpoint.line_start_offset; self.emit_heredoc_body_tokens = checkpoint.emit_heredoc_body_tokens; - self.current_quote_op = checkpoint.current_quote_op.as_ref().map(|quote| { - QuoteOperatorInfo { + self.current_quote_op = + checkpoint.current_quote_op.as_ref().map(|quote| QuoteOperatorInfo { operator: quote.operator.clone(), delimiter: quote.delimiter, start_pos: quote.start_pos, - } - }); + }); self.qw_recovery_enabled = checkpoint.qw_recovery_enabled; self.eof_emitted = checkpoint.eof_emitted; @@ -114,8 +111,8 @@ impl Checkpointable for PerlLexer<'_> { #[cfg(test)] mod tests { use super::*; - use crate::checkpoint::CheckpointContext; use crate::Position; + use crate::checkpoint::CheckpointContext; type TestResult = std::result::Result<(), String>; @@ -203,18 +200,12 @@ mod tests { lexer.paren_depth = 4; lexer.current_pos = Position { byte: 32, line: 3, column: 5 }; lexer.after_newline = false; - lexer.pending_heredocs = vec![HeredocSpec { - label: Arc::from("END"), - body_start: 48, - allow_indent: true, - }]; + lexer.pending_heredocs = + vec![HeredocSpec { label: Arc::from("END"), body_start: 48, allow_indent: true }]; lexer.line_start_offset = 24; lexer.emit_heredoc_body_tokens = true; - lexer.current_quote_op = Some(QuoteOperatorInfo { - operator: "s".to_string(), - delimiter: '{', - start_pos: 28, - }); + lexer.current_quote_op = + Some(QuoteOperatorInfo { operator: "s".to_string(), delimiter: '{', start_pos: 28 }); lexer.qw_recovery_enabled = false; lexer.eof_emitted = false; diff --git a/crates/perl-parser/src/incremental/lex.rs b/crates/perl-parser/src/incremental/lex.rs index e8351d70fc..dbe20acea0 100644 --- a/crates/perl-parser/src/incremental/lex.rs +++ b/crates/perl-parser/src/incremental/lex.rs @@ -12,10 +12,7 @@ pub(crate) struct LexedSource { pub(crate) live_checkpoints: Vec, } -fn summarize_checkpoint( - checkpoint: &LiveLexerCheckpoint, - line_index: &LineIndex, -) -> LexCheckpoint { +fn summarize_checkpoint(checkpoint: &LiveLexerCheckpoint, line_index: &LineIndex) -> LexCheckpoint { let (line, column) = line_index.byte_to_position(checkpoint.position); LexCheckpoint { byte: checkpoint.position, mode: checkpoint.mode, line, column } } @@ -83,6 +80,9 @@ pub(crate) fn capture_live_checkpoint( loop { let checkpoint = lexer.checkpoint(); if checkpoint.position == boundary { + if checkpoint.is_timeout_sensitive() { + return None; + } return Some(checkpoint); } if checkpoint.position > boundary { @@ -194,7 +194,9 @@ mod tests { let replayed = capture_live_checkpoint(source, expected.position) .ok_or_else(|| anyhow::anyhow!("live checkpoint replay failed"))?; - assert_eq!(&replayed, expected); + let mut expected_without_operation_time = expected.clone(); + expected_without_operation_time.start_time = replayed.start_time; + assert_eq!(replayed, expected_without_operation_time); Ok(()) } @@ -219,9 +221,10 @@ mod tests { lexed.checkpoints.iter().any(|summary| summary.byte == queued.position), "queued-heredoc state must remain a replayable boundary" ); - let replayed = capture_live_checkpoint(source, queued.position) - .ok_or_else(|| anyhow::anyhow!("queued-heredoc checkpoint replay failed"))?; - assert_eq!(replayed, *queued); + assert!( + capture_live_checkpoint(source, queued.position).is_none(), + "timeout-sensitive queued-heredoc checkpoints must fall back" + ); let resumed = lexed .live_checkpoints diff --git a/crates/perl-parser/tests/incremental_lexer_restart.rs b/crates/perl-parser/tests/incremental_lexer_restart.rs index 1d0e2de711..19fb02b4f3 100644 --- a/crates/perl-parser/tests/incremental_lexer_restart.rs +++ b/crates/perl-parser/tests/incremental_lexer_restart.rs @@ -102,12 +102,7 @@ fn stateful_and_source_boundary_edits_match_fresh_lexing() -> TestResult { ("prototype", "sub f($$) { return 1; } my $after = 1;", "return 1", "return 2"), ("unicode", "my $x = \"café\"; my $after = 1;", "é", "ø"), ("crlf", "my $x = 1;\r\nmy $y = 2;\r\n", "= 2", "= 3"), - ( - "heredoc-body", - "my $value = < TestResult { assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); Ok(()) } + +#[test] +fn timeout_sensitive_checkpoint_falls_back_with_downstream_span_parity() -> TestResult { + let source = "my $value = <= downstream_start).collect::>(); + let expected_downstream = + expected.iter().filter(|token| token.start >= downstream_start).collect::>(); + assert_eq!(actual_downstream.len(), expected_downstream.len()); + for (index, (actual, expected)) in actual_downstream.iter().zip(expected_downstream).enumerate() + { + assert_eq!(actual.token_type, expected.token_type, "downstream token kind {index}"); + assert_eq!(actual.text, expected.text, "downstream token payload {index}"); + assert_eq!(actual.start, expected.start, "downstream token start {index}"); + assert_eq!(actual.end, expected.end, "downstream token end {index}"); + } + Ok(()) +} From 847a25fb36043d7c8b60f7a83ba6c1d50f9c9cfd Mon Sep 17 00:00:00 2001 From: "Steven Zimmerman, CPA" <15812269+EffortlessSteven@users.noreply.github.com> Date: Wed, 12 Aug 2026 01:47:16 -0400 Subject: [PATCH 46/48] feat(parser): persist complete lexer restart checkpoints --- .../src/incremental/diagnostics.rs | 27 +- crates/perl-parser/src/incremental/lex.rs | 344 ++++++++++++------ crates/perl-parser/src/incremental/mod.rs | 5 +- crates/perl-parser/src/incremental/reparse.rs | 119 +++--- crates/perl-parser/src/incremental/state.rs | 70 ++-- .../tests/incremental_lexer_restart.rs | 33 +- 6 files changed, 370 insertions(+), 228 deletions(-) diff --git a/crates/perl-parser/src/incremental/diagnostics.rs b/crates/perl-parser/src/incremental/diagnostics.rs index 18b9d062a3..00a3e07957 100644 --- a/crates/perl-parser/src/incremental/diagnostics.rs +++ b/crates/perl-parser/src/incremental/diagnostics.rs @@ -10,8 +10,11 @@ pub enum LexRestartStrategy { Unchanged, /// Lex the complete current source from byte zero. FullRelex, - /// Restore one complete live lexer checkpoint and re-lex from there to EOF. + /// Restore a complete checkpoint reproduced by replaying the old prefix. + #[deprecated(note = "Use StoredCheckpointToEof; replay is no longer the canonical path.")] LiveCheckpointToEof, + /// Restore a complete generation-bound checkpoint without replaying old bytes. + StoredCheckpointToEof, } /// Truthful lexer restart and token-retention receipt. @@ -21,16 +24,17 @@ pub struct LexRestartReport { /// Strategy that produced the current token stream. pub strategy: LexRestartStrategy, /// Byte boundary where fresh lexing began. - /// - /// For [`LexRestartStrategy::Unchanged`], this is the current source length: - /// the complete old token stream is retained and no byte is freshly lexed. pub restart_byte: usize, - /// Number of source bytes lexed from the restart boundary to EOF. + /// Number of old-source prefix bytes replayed only to reconstruct state. + pub old_prefix_bytes_replayed: usize, + /// Number of current-source bytes lexed from restart to EOF. pub relexed_bytes: usize, /// Tokens before the restart boundary retained without re-lexing. pub reused_prefix_tokens: usize, /// Tokens after a synchronization boundary retained from the old suffix. pub reused_suffix_tokens: usize, + /// Complete generation-bound checkpoints retained in the resulting state. + pub stored_checkpoint_count: usize, } impl LexRestartReport { @@ -48,26 +52,15 @@ pub struct ReparseResult { /// Byte ranges reparsed or replaced by the selected strategy. pub changed_ranges: Vec>, /// Authoritative native parser output for the current source generation. - /// - /// This carries the AST, ordered parser diagnostics, recovery count, - /// budget usage, and early-termination state produced by the same - /// `Parser::parse_with_recovery` contract used by a fresh parse. pub parse_output: ParseOutput, /// Legacy LSP-shaped diagnostics retained for compatibility. - /// - /// Parser consumers should use [`Self::parse_output`]. LSP projection is a - /// transport concern and remains intentionally separate from the native - /// parser output contract. pub diagnostics: Vec, /// Lexer restart, fresh-work, and token-retention receipt. pub lex_restart: LexRestartReport, /// Number of source bytes covered by parser reparsing work. pub reparsed_bytes: usize, /// Compatibility total of old lexer tokens retained from prefix and suffix. - /// - /// New consumers should use [`Self::lex_restart`] to distinguish prefix - /// retention from state-proven suffix reuse. pub reused_tokens: usize, /// Total token count in the resulting incremental state. pub token_count: usize, -} +} \ No newline at end of file diff --git a/crates/perl-parser/src/incremental/lex.rs b/crates/perl-parser/src/incremental/lex.rs index dbe20acea0..fbd3952156 100644 --- a/crates/perl-parser/src/incremental/lex.rs +++ b/crates/perl-parser/src/incremental/lex.rs @@ -1,17 +1,113 @@ use crate::incremental::LineIndex; use crate::incremental::checkpoint::LexCheckpoint; +use crate::incremental::edit::Edit; use anyhow::Result; use perl_lexer::{ Checkpointable, LexerCheckpoint as LiveLexerCheckpoint, PerlLexer, Token, TokenType, }; +/// Hard cap for complete behavior-bearing checkpoints retained in one state. +pub const MAX_STORED_LEX_CHECKPOINTS: usize = 4096; +const INITIAL_CHECKPOINT_SPACING: usize = 128; + +/// One complete checkpoint bound to the exact source generation that produced it. +#[derive(Clone)] +pub(crate) struct StoredLexCheckpoint { + pub(crate) summary: LexCheckpoint, + pub(crate) live: LiveLexerCheckpoint, + source_fingerprint: u64, + prefix_fingerprint: u64, +} + +impl StoredLexCheckpoint { + fn capture(source: &str, live: LiveLexerCheckpoint, line_index: &LineIndex) -> Self { + let summary = summarize_checkpoint(&live, line_index); + let prefix = source.get(..live.position).unwrap_or_default(); + Self { + summary, + live, + source_fingerprint: fingerprint(source.as_bytes()), + prefix_fingerprint: fingerprint(prefix.as_bytes()), + } + } + + fn belongs_to_source(&self, source: &str) -> bool { + if self.live.position > source.len() || !source.is_char_boundary(self.live.position) { + return false; + } + let Some(prefix) = source.get(..self.live.position) else { + return false; + }; + self.source_fingerprint == fingerprint(source.as_bytes()) + && self.prefix_fingerprint == fingerprint(prefix.as_bytes()) + && self.live.is_valid_for(source) + } + + /// Clone and transform the complete state for one old-generation edit. + pub(crate) fn prepare_for_edit( + &self, + old_source: &str, + edit: &Edit, + ) -> Option { + if !self.belongs_to_source(old_source) + || self.summary.byte > edit.start_byte + || self.live.is_timeout_sensitive() + { + return None; + } + let old_len = edit.old_end_byte.checked_sub(edit.start_byte)?; + let mut live = self.live.clone(); + live.try_apply_edit(edit.start_byte, old_len, edit.new_text.len()).then_some(live) + } + + /// Carry an old prefix checkpoint into the edited generation when all + /// behavior-bearing offsets and prefix bytes remain valid. + pub(crate) fn transform_for_generation( + &self, + old_source: &str, + new_source: &str, + edit: &Edit, + ) -> Option { + if !self.belongs_to_source(old_source) || self.summary.byte >= edit.start_byte { + return None; + } + let old_len = edit.old_end_byte.checked_sub(edit.start_byte)?; + let mut live = self.live.clone(); + if !live.try_apply_edit(edit.start_byte, old_len, edit.new_text.len()) + || !live.is_valid_for(new_source) + { + return None; + } + let prefix = new_source.get(..live.position)?; + if fingerprint(prefix.as_bytes()) != self.prefix_fingerprint { + return None; + } + Some(Self { + summary: self.summary, + live, + source_fingerprint: fingerprint(new_source.as_bytes()), + prefix_fingerprint: self.prefix_fingerprint, + }) + } +} + pub(crate) struct LexedSource { pub(crate) tokens: Vec, pub(crate) checkpoints: Vec, + pub(crate) stored_checkpoints: Vec, #[cfg(test)] pub(crate) live_checkpoints: Vec, } +fn fingerprint(bytes: &[u8]) -> u64 { + let mut value = 0xcbf2_9ce4_8422_2325_u64; + for byte in bytes { + value ^= u64::from(*byte); + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + value +} + fn summarize_checkpoint(checkpoint: &LiveLexerCheckpoint, line_index: &LineIndex) -> LexCheckpoint { let (line, column) = line_index.byte_to_position(checkpoint.position); LexCheckpoint { byte: checkpoint.position, mode: checkpoint.mode, line, column } @@ -22,30 +118,88 @@ fn push_summary( checkpoint: &LiveLexerCheckpoint, line_index: &LineIndex, ) { - // A queued/virtual lexer event may expose several internal states at one - // byte. The public summary is one replayable boundary per byte and maps to - // the first complete live state reproduced by `capture_live_checkpoint`. if summaries.last().is_none_or(|summary| summary.byte != checkpoint.position) { summaries.push(summarize_checkpoint(checkpoint, line_index)); } } -/// Lex one complete source and capture restart candidates from the lexer's -/// actual mutable state before emitted tokens and terminal EOF. -/// -/// The live checkpoint includes the ordered heredoc queue, newline/line-start -/// context, quote-operator state, body-emission policy, and recovery policy, so -/// heredoc boundaries no longer need a parser-side suppression approximation. +fn behavior_state_changed( + previous: &LiveLexerCheckpoint, + current: &LiveLexerCheckpoint, +) -> bool { + previous.mode != current.mode + || previous.delimiter_stack != current.delimiter_stack + || previous.in_prototype != current.in_prototype + || previous.prototype_depth != current.prototype_depth + || previous.after_sub != current.after_sub + || previous.after_arrow != current.after_arrow + || previous.hash_brace_depth != current.hash_brace_depth + || previous.after_var_subscript != current.after_var_subscript + || previous.paren_depth != current.paren_depth + || previous.after_newline != current.after_newline + || previous.pending_heredocs != current.pending_heredocs + || previous.line_start_offset != current.line_start_offset + || previous.emit_heredoc_body_tokens != current.emit_heredoc_body_tokens + || previous.current_quote_op != current.current_quote_op + || previous.qw_recovery_enabled != current.qw_recovery_enabled + || previous.eof_emitted != current.eof_emitted + || previous.context != current.context +} + +fn compact_stored_checkpoints(checkpoints: &mut Vec) { + let original = std::mem::take(checkpoints); + let original_len = original.len(); + checkpoints.extend(original.into_iter().enumerate().filter_map(|(index, checkpoint)| { + (index == 0 || index % 2 == 0 || index + 1 == original_len).then_some(checkpoint) + })); +} + +fn push_stored_checkpoint( + source: &str, + line_index: &LineIndex, + checkpoints: &mut Vec, + spacing: &mut usize, + live: &LiveLexerCheckpoint, +) { + if checkpoints.last().is_some_and(|stored| stored.live.position == live.position) { + return; + } + + let retain = checkpoints.last().is_none_or(|previous| { + live.position.saturating_sub(previous.live.position) >= *spacing + || behavior_state_changed(&previous.live, live) + }); + if !retain { + return; + } + + if checkpoints.len() >= MAX_STORED_LEX_CHECKPOINTS { + compact_stored_checkpoints(checkpoints); + *spacing = spacing.saturating_mul(2).max(1); + } + checkpoints.push(StoredLexCheckpoint::capture(source, live.clone(), line_index)); +} + +/// Lex one complete source and capture complete, generation-bound restart state. pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) -> LexedSource { let mut lexer = PerlLexer::new(source); let mut tokens = Vec::new(); let mut checkpoints = Vec::new(); + let mut stored_checkpoints = Vec::new(); + let mut checkpoint_spacing = INITIAL_CHECKPOINT_SPACING; #[cfg(test)] let mut live_checkpoints = Vec::new(); loop { let live = lexer.checkpoint(); push_summary(&mut checkpoints, &live, line_index); + push_stored_checkpoint( + source, + line_index, + &mut stored_checkpoints, + &mut checkpoint_spacing, + &live, + ); #[cfg(test)] live_checkpoints.push(live); @@ -53,6 +207,15 @@ pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) break; }; if token.token_type == TokenType::EOF { + let terminal = lexer.checkpoint(); + push_summary(&mut checkpoints, &terminal, line_index); + push_stored_checkpoint( + source, + line_index, + &mut stored_checkpoints, + &mut checkpoint_spacing, + &terminal, + ); break; } tokens.push(token); @@ -61,44 +224,13 @@ pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) LexedSource { tokens, checkpoints, + stored_checkpoints, #[cfg(test)] live_checkpoints, } } -/// Replay the old source to one previously captured token boundary and return -/// the complete current `Checkpointable` state for that exact boundary. -/// -/// The public `LexCheckpoint` remains a compact compatibility summary. Restart -/// correctness is authorized only by the full state returned here. -pub(crate) fn capture_live_checkpoint( - source: &str, - boundary: usize, -) -> Option { - let mut lexer = PerlLexer::new(source); - - loop { - let checkpoint = lexer.checkpoint(); - if checkpoint.position == boundary { - if checkpoint.is_timeout_sensitive() { - return None; - } - return Some(checkpoint); - } - if checkpoint.position > boundary { - return None; - } - - match lexer.next_token() { - Some(token) if token.token_type != TokenType::EOF => {} - _ => return None, - } - } -} - -/// Restore the complete mutable checkpoint contract into the edited source and -/// re-lex from that boundary to EOF. No old suffix is reused in this -/// correctness-first strategy. +/// Restore complete mutable state into the edited source and re-lex to EOF. pub(crate) fn lex_from_live_checkpoint( source: &str, line_index: &LineIndex, @@ -106,12 +238,14 @@ pub(crate) fn lex_from_live_checkpoint( ) -> Result { let mut lexer = PerlLexer::new(source); if !lexer.can_restore(checkpoint) { - anyhow::bail!("live lexer checkpoint is not valid for the edited source"); + anyhow::bail!("stored lexer checkpoint is not valid for the edited source"); } lexer.restore(checkpoint); let mut tokens = Vec::new(); let mut checkpoints = Vec::new(); + let mut stored_checkpoints = Vec::new(); + let mut checkpoint_spacing = INITIAL_CHECKPOINT_SPACING; #[cfg(test)] let mut live_checkpoints = Vec::new(); let mut last_position = checkpoint.position; @@ -119,6 +253,13 @@ pub(crate) fn lex_from_live_checkpoint( loop { let live = lexer.checkpoint(); push_summary(&mut checkpoints, &live, line_index); + push_stored_checkpoint( + source, + line_index, + &mut stored_checkpoints, + &mut checkpoint_spacing, + &live, + ); #[cfg(test)] live_checkpoints.push(live); @@ -126,6 +267,15 @@ pub(crate) fn lex_from_live_checkpoint( break; }; if token.token_type == TokenType::EOF { + let terminal = lexer.checkpoint(); + push_summary(&mut checkpoints, &terminal, line_index); + push_stored_checkpoint( + source, + line_index, + &mut stored_checkpoints, + &mut checkpoint_spacing, + &terminal, + ); break; } if token.end <= last_position { @@ -138,6 +288,7 @@ pub(crate) fn lex_from_live_checkpoint( Ok(LexedSource { tokens, checkpoints, + stored_checkpoints, #[cfg(test)] live_checkpoints, }) @@ -147,6 +298,42 @@ pub(crate) fn lex_from_live_checkpoint( mod tests { use super::*; + #[test] + fn stored_checkpoint_rejects_a_different_source_generation() -> Result<()> { + let source = "my $value = 1;"; + let line_index = LineIndex::new(source); + let lexed = lex_source_with_checkpoints(source, &line_index); + let stored = lexed + .stored_checkpoints + .first() + .ok_or_else(|| anyhow::anyhow!("origin checkpoint is missing"))?; + let edit = Edit { + start_byte: source.len(), + old_end_byte: source.len(), + new_end_byte: source.len(), + new_text: String::new(), + }; + + assert!(stored.prepare_for_edit("my $value = 2;", &edit).is_none()); + Ok(()) + } + + #[test] + fn stored_checkpoint_set_is_bounded_and_retains_a_late_boundary() -> Result<()> { + let source = (0..20_000).map(|index| format!("my $v{index} = {index};\n")).collect::(); + let line_index = LineIndex::new(&source); + let lexed = lex_source_with_checkpoints(&source, &line_index); + + assert!(!lexed.stored_checkpoints.is_empty()); + assert!(lexed.stored_checkpoints.len() <= MAX_STORED_LEX_CHECKPOINTS); + let last = lexed + .stored_checkpoints + .last() + .ok_or_else(|| anyhow::anyhow!("last checkpoint is missing"))?; + assert!(last.summary.byte > source.len() / 2); + Ok(()) + } + #[test] fn live_checkpoint_preserves_after_arrow_state() -> Result<()> { let source = "$object->method();"; @@ -170,73 +357,22 @@ mod tests { let line_index = LineIndex::new(source); let lexed = lex_source_with_checkpoints(source, &line_index); - assert!( - lexed.live_checkpoints.iter().any(|checkpoint| checkpoint.in_prototype), - "prototype fixture must expose an in_prototype checkpoint" - ); - assert!( - lexed.live_checkpoints.iter().any(|checkpoint| checkpoint.paren_depth > 0), - "prototype fixture must expose parenthesis depth" - ); - } - - #[test] - fn replay_captures_the_same_complete_checkpoint() -> Result<()> { - let source = "$object->method();"; - let line_index = LineIndex::new(source); - let lexed = lex_source_with_checkpoints(source, &line_index); - let method_index = lexed - .tokens - .iter() - .position(|token| token.text.as_ref() == "method") - .ok_or_else(|| anyhow::anyhow!("method token is missing"))?; - let expected = &lexed.live_checkpoints[method_index]; - let replayed = capture_live_checkpoint(source, expected.position) - .ok_or_else(|| anyhow::anyhow!("live checkpoint replay failed"))?; - - let mut expected_without_operation_time = expected.clone(); - expected_without_operation_time.start_time = replayed.start_time; - assert_eq!(replayed, expected_without_operation_time); - Ok(()) + assert!(lexed.live_checkpoints.iter().any(|checkpoint| checkpoint.in_prototype)); + assert!(lexed.live_checkpoints.iter().any(|checkpoint| checkpoint.paren_depth > 0)); } #[test] - fn heredoc_queue_is_captured_and_restart_boundaries_resume() -> Result<()> { + fn heredoc_queue_is_captured_but_timeout_sensitive_state_is_not_selected() -> Result<()> { let source = "my $value = < queued.position && checkpoint.pending_heredocs.is_empty() - }) - .ok_or_else(|| anyhow::anyhow!("checkpoint after heredoc completion is missing"))?; - assert!( - lexed.checkpoints.iter().any(|summary| summary.byte == resumed.position), - "restart summaries must resume after the heredoc queue drains" - ); + assert!(lexed.live_checkpoints.iter().any(|checkpoint| { + !checkpoint.pending_heredocs.is_empty() && checkpoint.is_timeout_sensitive() + })); + assert!(lexed.stored_checkpoints.iter().any(|checkpoint| { + !checkpoint.live.pending_heredocs.is_empty() + })); Ok(()) } -} +} \ No newline at end of file diff --git a/crates/perl-parser/src/incremental/mod.rs b/crates/perl-parser/src/incremental/mod.rs index 31cd9943af..211aa5d6da 100644 --- a/crates/perl-parser/src/incremental/mod.rs +++ b/crates/perl-parser/src/incremental/mod.rs @@ -17,6 +17,7 @@ pub use perl_line_index::LineIndex; pub use checkpoint::{LexCheckpoint, ParseCheckpoint, ScopeSnapshot}; pub use diagnostics::{LexRestartReport, LexRestartStrategy, ReparseResult}; pub use edit::Edit; +pub use lex::MAX_STORED_LEX_CHECKPOINTS; use reparse::{apply_single_edit, apply_text_edit_to_state, full_reparse}; pub use state::IncrementalState; pub use strategy::MAX_EDIT_SIZE; @@ -95,9 +96,11 @@ fn unchanged_result(state: &IncrementalState) -> ReparseResult { let lex_restart = LexRestartReport { strategy: LexRestartStrategy::Unchanged, restart_byte: state.source().len(), + old_prefix_bytes_replayed: 0, relexed_bytes: 0, reused_prefix_tokens: state.tokens().len(), reused_suffix_tokens: 0, + stored_checkpoint_count: state.stored_lex_checkpoint_count(), }; ReparseResult { changed_ranges: Vec::new(), @@ -175,4 +178,4 @@ pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result Result { - let summary = state - .find_lex_checkpoint(edit.start_byte) - .copied() - .ok_or_else(|| anyhow::anyhow!("No lexer restart boundary found"))?; - let mut live_checkpoint = capture_live_checkpoint(state.source(), summary.byte) - .ok_or_else(|| anyhow::anyhow!("Could not reproduce complete live lexer state at restart boundary"))?; - let old_len = edit - .old_end_byte - .checked_sub(edit.start_byte) - .ok_or_else(|| anyhow::anyhow!("edit end precedes edit start"))?; - if !live_checkpoint.try_apply_edit(edit.start_byte, old_len, edit.new_text.len()) { - anyhow::bail!("Edit invalidated required live lexer state"); - } - + let old_source = state.source().to_string(); + let selected = state + .stored_lex_checkpoints() + .iter() + .rev() + .filter(|stored| stored.summary.byte <= edit.start_byte) + .find_map(|stored| { + stored + .prepare_for_edit(&old_source, edit) + .map(|live| (stored.summary, live)) + }) + .ok_or_else(|| anyhow::anyhow!("No valid stored lexer checkpoint found"))?; + let (summary, live_checkpoint) = selected; let restart_byte = live_checkpoint.position; let reused_prefix_tokens = state .tokens() .iter() .take_while(|token| token.start < restart_byte) .count(); - apply_text_edit_to_state(state, edit)?; + let old_prefix_checkpoints = state + .stored_lex_checkpoints() + .iter() + .take_while(|checkpoint| checkpoint.summary.byte < restart_byte) + .cloned() + .collect::>(); + apply_text_edit_to_state(state, edit)?; let lexed = lex_from_live_checkpoint(state.source(), state.line_index(), &live_checkpoint)?; let mut tokens = state.tokens()[..reused_prefix_tokens].to_vec(); tokens.extend(lexed.tokens); - let mut checkpoints = state + let mut checkpoint_summaries = state .lex_checkpoints() .iter() .take_while(|checkpoint| checkpoint.byte < restart_byte) .copied() .collect::>(); - checkpoints.extend(lexed.checkpoints); - state.replace_lex_state(tokens, checkpoints); + checkpoint_summaries.extend(lexed.checkpoints); + + let mut stored_checkpoints = old_prefix_checkpoints + .iter() + .filter_map(|checkpoint| { + checkpoint.transform_for_generation(&old_source, state.source(), edit) + }) + .collect::>(); + stored_checkpoints.extend(lexed.stored_checkpoints); + + state.replace_lex_state(tokens, checkpoint_summaries, stored_checkpoints); let lex_restart = LexRestartReport { - strategy: LexRestartStrategy::LiveCheckpointToEof, + strategy: LexRestartStrategy::StoredCheckpointToEof, restart_byte, + old_prefix_bytes_replayed: 0, relexed_bytes: state.source().len().saturating_sub(restart_byte), reused_prefix_tokens, reused_suffix_tokens: 0, + stored_checkpoint_count: state.stored_lex_checkpoint_count(), }; + debug_assert_eq!(summary.byte, restart_byte); Ok(SingleEditReparse { range: restart_byte..state.source().len(), lex_restart, @@ -89,14 +106,16 @@ pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result Result<()> { + fn equal_width_edit_restores_stored_state_without_replaying_old_bytes() -> Result<()> { let source = "my $x = 1; my $y = 2;"; let start = source.find("= 1").ok_or_else(|| anyhow::anyhow!("literal missing"))? + 2; let edit = Edit { @@ -150,7 +169,8 @@ mod tests { let mut state = IncrementalState::new(source.to_string()); let result = apply_single_edit(&mut state, &edit)?; - assert_eq!(result.lex_restart.strategy, LexRestartStrategy::LiveCheckpointToEof); + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); assert_eq!(result.lex_restart.reused_suffix_tokens, 0); assert_eq!(result.range.end, state.source().len()); assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); @@ -158,7 +178,7 @@ mod tests { } #[test] - fn method_name_edit_restores_after_arrow_state_and_matches_fresh_lex() -> Result<()> { + fn method_name_edit_matches_fresh_lex_from_stored_state() -> Result<()> { let source = "$object->method(); my $x = 1;"; let start = source.find("method").ok_or_else(|| anyhow::anyhow!("method missing"))?; let edit = Edit { @@ -170,13 +190,13 @@ mod tests { let mut state = IncrementalState::new(source.to_string()); let result = apply_single_edit(&mut state, &edit)?; - assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); Ok(()) } #[test] - fn heredoc_body_edit_restores_the_live_queue_and_matches_fresh_lex() -> Result<()> { + fn heredoc_body_edit_uses_an_earlier_safe_stored_checkpoint() -> Result<()> { let source = "my $value = < Result<()> { - let source = "my $a = 1;\nmy $b = 2;\nmy $c = 3;\n"; - let delete_len = "my $a = 1;\n".len(); - let edit = Edit { - start_byte: 0, - old_end_byte: delete_len, - new_end_byte: 0, - new_text: String::new(), - }; + fn sequential_edits_regenerate_current_generation_checkpoints() -> Result<()> { + let source = "my $a = 1; my $b = 2; my $c = 3;"; let mut state = IncrementalState::new(source.to_string()); - let result = apply_single_edit(&mut state, &edit)?; + let first_start = source.find("= 2").ok_or_else(|| anyhow::anyhow!("first edit missing"))? + 2; + let first = Edit { + start_byte: first_start, + old_end_byte: first_start + 1, + new_end_byte: first_start + 1, + new_text: "8".to_string(), + }; + apply_single_edit(&mut state, &first)?; + + let second_start = state + .source() + .find("= 3") + .ok_or_else(|| anyhow::anyhow!("second edit missing"))? + 2; + let second = Edit { + start_byte: second_start, + old_end_byte: second_start + 1, + new_end_byte: second_start + 1, + new_text: "9".to_string(), + }; + let result = apply_single_edit(&mut state, &second)?; - assert_eq!(result.lex_restart.restart_byte, 0); - assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); - for token in state.tokens() { - assert!(token.start <= token.end); - assert!(token.end <= state.source().len()); - assert!(state.source().is_char_boundary(token.start)); - assert!(state.source().is_char_boundary(token.end)); - } Ok(()) } -} +} \ No newline at end of file diff --git a/crates/perl-parser/src/incremental/state.rs b/crates/perl-parser/src/incremental/state.rs index b34c5a42a1..a55af8dbc0 100644 --- a/crates/perl-parser/src/incremental/state.rs +++ b/crates/perl-parser/src/incremental/state.rs @@ -1,5 +1,5 @@ use crate::incremental::checkpoint::{LexCheckpoint, ParseCheckpoint, ScopeSnapshot}; -use crate::incremental::lex::lex_source_with_checkpoints; +use crate::incremental::lex::{StoredLexCheckpoint, lex_source_with_checkpoints}; use perl_lexer::Token; use perl_line_index::LineIndex; use perl_parser_core::ast::{Node, NodeKind}; @@ -9,11 +9,6 @@ use ropey::Rope; use std::ops::Deref; /// Read-only compatibility view for legacy field-style access. -/// -/// `IncrementalState` intentionally implements `Deref` but not `DerefMut` for -/// this view. Existing consumers may continue to read `state.source` and -/// `state.lex_checkpoints`, while generation replacement remains private to the -/// committed-state machinery. #[doc(hidden)] #[derive(Clone)] pub struct IncrementalStateReadView { @@ -25,37 +20,10 @@ pub struct IncrementalStateReadView { /// One internally consistent incremental parser generation. /// -/// Generation-bearing fields are crate-private so external callers cannot -/// mutate source, tokens, checkpoints, AST, or parser output independently. -/// Use [`IncrementalState::new`], read-only accessors, and [`super::apply_edits`] -/// to move between committed generations. -/// -/// Legacy field-style reads remain available through an immutable compatibility -/// view: -/// -/// ``` -/// use perl_parser::incremental::IncrementalState; -/// -/// let state = IncrementalState::new("my $x = 1;".to_string()); -/// assert_eq!(state.source.len(), state.source().len()); -/// assert_eq!(state.lex_checkpoints.len(), state.lex_checkpoints().len()); -/// ``` -/// -/// The view does not grant mutation authority: -/// -/// ```compile_fail -/// use perl_parser::incremental::IncrementalState; -/// -/// let mut state = IncrementalState::new("my $x = 1;".to_string()); -/// state.source.push_str("\n"); -/// ``` -/// -/// ```compile_fail -/// use perl_parser::incremental::IncrementalState; -/// -/// let mut state = IncrementalState::new("my $x = 1;".to_string()); -/// state.lex_checkpoints.clear(); -/// ``` +/// Complete behavior-bearing lexer checkpoints are retained privately and move +/// atomically with source, token, line-index, and parser state. The public +/// `LexCheckpoint` vector remains a compact compatibility summary; it is not the +/// restart authority. #[derive(Clone)] #[non_exhaustive] pub struct IncrementalState { @@ -68,6 +36,7 @@ pub struct IncrementalState { #[deprecated(note = "Use parse_output(); this compatibility mirror will be removed.")] pub(super) ast: Node, pub(super) tokens: Vec, + pub(super) stored_lex_checkpoints: Vec, pub(super) read_view: IncrementalStateReadView, } @@ -92,6 +61,7 @@ impl IncrementalState { let lexed = lex_source_with_checkpoints(&source, &line_index); let tokens = lexed.tokens; let lex_checkpoints = lexed.checkpoints; + let stored_lex_checkpoints = lexed.stored_checkpoints; let parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast); Self { rope, @@ -100,6 +70,7 @@ impl IncrementalState { parse_output, ast, tokens, + stored_lex_checkpoints, read_view: IncrementalStateReadView { source, lex_checkpoints }, } } @@ -128,6 +99,16 @@ impl IncrementalState { &self.read_view.lex_checkpoints } + /// Number of complete generation-bound lexer checkpoints retained privately. + #[must_use] + pub fn stored_lex_checkpoint_count(&self) -> usize { + self.stored_lex_checkpoints.len() + } + + pub(crate) fn stored_lex_checkpoints(&self) -> &[StoredLexCheckpoint] { + &self.stored_lex_checkpoints + } + /// Parser restart summaries for the current committed parse output. #[must_use] pub fn parse_checkpoints(&self) -> &[ParseCheckpoint] { @@ -153,7 +134,7 @@ impl IncrementalState { &self.tokens } - /// Find the nearest lexer checkpoint at or before `byte`. + /// Find the nearest compact lexer checkpoint at or before `byte`. #[must_use] pub fn find_lex_checkpoint(&self, byte: usize) -> Option<&LexCheckpoint> { self.read_view.lex_checkpoints.iter().rev().find(|cp| cp.byte <= byte) @@ -166,30 +147,25 @@ impl IncrementalState { } /// Replace the text-bearing portion of a staged generation. - /// - /// This remains crate-private so source, rope, and line-index identity cannot - /// be changed independently by consumers. Callers finish rebuilding tokens, - /// checkpoints, and parser output before publishing the staged state. pub(super) fn replace_source_text(&mut self, source: String) { self.rope = Rope::from_str(&source); self.line_index = LineIndex::new(&source); self.read_view.source = source; } - /// Replace the staged lexer output and its restart summaries together. + /// Replace the staged lexer output and both checkpoint planes together. pub(super) fn replace_lex_state( &mut self, tokens: Vec, lex_checkpoints: Vec, + stored_lex_checkpoints: Vec, ) { self.tokens = tokens; self.read_view.lex_checkpoints = lex_checkpoints; + self.stored_lex_checkpoints = stored_lex_checkpoints; } /// Refresh the authoritative parser output from the current source. - /// - /// The compatibility AST and parse checkpoints are updated from the same - /// recovered parse so the state cannot expose mixed parse generations. #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] pub(crate) fn refresh_parse_output(&mut self) { let mut parser = Parser::new(self.source()); @@ -254,4 +230,4 @@ fn walk_ast_for_checkpoints( } _ => {} } -} +} \ No newline at end of file diff --git a/crates/perl-parser/tests/incremental_lexer_restart.rs b/crates/perl-parser/tests/incremental_lexer_restart.rs index 19fb02b4f3..590b99f2f6 100644 --- a/crates/perl-parser/tests/incremental_lexer_restart.rs +++ b/crates/perl-parser/tests/incremental_lexer_restart.rs @@ -1,8 +1,8 @@ #![cfg(feature = "incremental")] -//! Public-contract tests for correctness-first live lexer restart. +//! Public-contract tests for generation-bound stored lexer restart. use perl_lexer::{PerlLexer, Token, TokenType}; -use perl_parser::incremental::LexRestartStrategy; +use perl_parser::incremental::{LexRestartStrategy, MAX_STORED_LEX_CHECKPOINTS}; use perl_parser::{Edit, IncrementalState, apply_edits}; type TestResult = Result<(), Box>; @@ -51,11 +51,13 @@ fn empty_edit_batch_reports_unchanged_without_lexer_or_parser_work() -> TestResu assert_eq!(result.lex_restart.strategy, LexRestartStrategy::Unchanged); assert_eq!(result.lex_restart.restart_byte, source.len()); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); assert_eq!(result.lex_restart.relexed_bytes, 0); assert_eq!(result.lex_restart.reused_prefix_tokens, token_count); assert_eq!(result.lex_restart.reused_suffix_tokens, 0); assert_eq!(result.reused_tokens, token_count); assert_eq!(result.reparsed_bytes, 0); + assert!(result.lex_restart.stored_checkpoint_count > 0); assert!(result.changed_ranges.is_empty()); assert_eq!(state.source(), source); assert_tokens_equal(state.tokens(), &fresh_tokens(source)); @@ -63,7 +65,7 @@ fn empty_edit_batch_reports_unchanged_without_lexer_or_parser_work() -> TestResu } #[test] -fn late_equal_width_edit_retains_prefix_and_relexes_the_complete_suffix() -> TestResult { +fn late_equal_width_edit_uses_stored_state_and_relexes_the_complete_suffix() -> TestResult { let source = "my $before = 1; my $target = 2; my $after = 3;"; let start = source.find("= 2").ok_or("target literal is missing")? + 2; let edit = Edit { @@ -75,7 +77,8 @@ fn late_equal_width_edit_retains_prefix_and_relexes_the_complete_suffix() -> Tes let mut state = IncrementalState::new(source.to_string()); let result = apply_edits(&mut state, &[edit])?; - assert_eq!(result.lex_restart.strategy, LexRestartStrategy::LiveCheckpointToEof); + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); assert!(result.lex_restart.restart_byte > 0); assert!(result.lex_restart.reused_prefix_tokens > 0); assert_eq!(result.lex_restart.reused_suffix_tokens, 0); @@ -112,9 +115,10 @@ fn stateful_and_source_boundary_edits_match_fresh_lexing() -> TestResult { assert_eq!( result.lex_restart.strategy, - LexRestartStrategy::LiveCheckpointToEof, - "{name} unexpectedly abandoned the live checkpoint path" + LexRestartStrategy::StoredCheckpointToEof, + "{name} unexpectedly abandoned the stored-checkpoint path" ); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0, "{name}"); assert_eq!(result.lex_restart.reused_suffix_tokens, 0, "{name}"); assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); } @@ -122,7 +126,7 @@ fn stateful_and_source_boundary_edits_match_fresh_lexing() -> TestResult { } #[test] -fn method_context_edit_matches_fresh_lexing_after_complete_state_restore() -> TestResult { +fn method_context_edit_matches_fresh_lexing_after_stored_state_restore() -> TestResult { let source = "my $value = $object->method(); my $after = 1;"; let start = source.find("method").ok_or("method name is missing")?; let edit = Edit { @@ -134,7 +138,8 @@ fn method_context_edit_matches_fresh_lexing_after_complete_state_restore() -> Te let mut state = IncrementalState::new(source.to_string()); let result = apply_edits(&mut state, &[edit])?; - assert_eq!(result.lex_restart.strategy, LexRestartStrategy::LiveCheckpointToEof); + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); assert_eq!(result.lex_restart.reused_suffix_tokens, 0); assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); Ok(()) @@ -155,24 +160,26 @@ fn large_edit_reports_full_relex_instead_of_checkpoint_reuse() -> TestResult { assert_eq!(result.lex_restart.strategy, LexRestartStrategy::FullRelex); assert_eq!(result.lex_restart.restart_byte, 0); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); assert_eq!(result.lex_restart.reused_prefix_tokens, 0); assert_eq!(result.lex_restart.reused_suffix_tokens, 0); assert_eq!(result.lex_restart.relexed_bytes, state.source().len()); + assert!(result.lex_restart.stored_checkpoint_count <= MAX_STORED_LEX_CHECKPOINTS); assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); Ok(()) } #[test] -fn timeout_sensitive_checkpoint_falls_back_with_downstream_span_parity() -> TestResult { +fn timeout_sensitive_heredoc_state_selects_an_earlier_safe_checkpoint() -> TestResult { let source = "my $value = < Test assert_eq!(actual.end, expected.end, "downstream token end {index}"); } Ok(()) -} +} \ No newline at end of file From b5d9af8498ca85f71e1f5c1ac3ac212f18e43b94 Mon Sep 17 00:00:00 2001 From: Steven Zimmerman Date: Wed, 12 Aug 2026 02:27:45 -0400 Subject: [PATCH 47/48] fix(parser): align checkpoint recovery with current lexer policy --- crates/perl-lexer/src/checkpoint/core.rs | 18 ++- crates/perl-lexer/src/checkpoint_impl.rs | 2 - crates/perl-lexer/src/lexer/driver.rs | 1 - crates/perl-lexer/src/lexer/state.rs | 3 - crates/perl-lexer/src/lib.rs | 19 +--- crates/perl-lexer/src/limits.rs | 1 - crates/perl-parser/src/incremental/lex.rs | 41 ++++++- .../tests/incremental_lexer_restart.rs | 106 ++++++++++++++---- 8 files changed, 132 insertions(+), 59 deletions(-) diff --git a/crates/perl-lexer/src/checkpoint/core.rs b/crates/perl-lexer/src/checkpoint/core.rs index eed5c2bbf4..f780aadd08 100644 --- a/crates/perl-lexer/src/checkpoint/core.rs +++ b/crates/perl-lexer/src/checkpoint/core.rs @@ -25,13 +25,11 @@ pub struct QuoteOperatorCheckpoint { /// A checkpoint that captures all mutable lexer state needed for token replay. /// -/// Input references are deliberately not persisted. The monotonic timeout -/// origin is retained so restoring a checkpoint cannot silently grant a fresh -/// heredoc timeout budget. +/// Input references are deliberately not persisted. Heredoc recovery uses the +/// deterministic byte budget configured by the lexer, and checkpoints that +/// already have queued heredocs are rejected by the incremental replay layer. #[derive(Debug, Clone, PartialEq)] pub struct LexerCheckpoint { - /// Monotonic origin used by timeout-sensitive lexer paths. - pub start_time: std::time::Instant, /// Current position in the input. pub position: usize, /// Current lexer mode (`ExpectTerm`, `ExpectOperator`, etc.). @@ -112,7 +110,6 @@ impl LexerCheckpoint { #[must_use] pub fn new() -> Self { Self { - start_time: std::time::Instant::now(), position: 0, mode: LexerMode::ExpectTerm, delimiter_stack: Vec::new(), @@ -154,9 +151,9 @@ impl LexerCheckpoint { self.position == 0 } - /// Whether restoring this checkpoint would re-enter a wall-clock-bounded - /// lexer path. Callers must fall back to a full re-lex when this is true - /// unless they can prove the timeout origin is safe for the operation. + /// Whether restoring this checkpoint would re-enter a pending-heredoc + /// byte-budget path. Incremental callers fail closed for this state so + /// heredoc recovery remains owned by a fresh full lex. #[must_use] pub fn is_timeout_sensitive(&self) -> bool { !self.pending_heredocs.is_empty() @@ -359,8 +356,7 @@ pub trait Checkpointable { /// Restore mutable replay state into a lexer for the target input. /// - /// The target lexer retains its configured policy and checkpoint timeout - /// origin. + /// The target lexer retains its configured deterministic recovery policy. fn restore(&mut self, checkpoint: &LexerCheckpoint); /// Check whether every source-relative checkpoint offset is valid. diff --git a/crates/perl-lexer/src/checkpoint_impl.rs b/crates/perl-lexer/src/checkpoint_impl.rs index 9e3a87e1ab..15c885a8ee 100644 --- a/crates/perl-lexer/src/checkpoint_impl.rs +++ b/crates/perl-lexer/src/checkpoint_impl.rs @@ -28,7 +28,6 @@ impl Checkpointable for PerlLexer<'_> { }; LexerCheckpoint { - start_time: self.start_time, position: self.position, mode: self.mode, delimiter_stack: self.delimiter_stack.clone(), @@ -64,7 +63,6 @@ impl Checkpointable for PerlLexer<'_> { } fn restore(&mut self, checkpoint: &LexerCheckpoint) { - self.start_time = checkpoint.start_time; self.position = checkpoint.position; self.mode = checkpoint.mode; self.delimiter_stack.clone_from(&checkpoint.delimiter_stack); diff --git a/crates/perl-lexer/src/lexer/driver.rs b/crates/perl-lexer/src/lexer/driver.rs index f7d2c0bc4d..df455b8f89 100644 --- a/crates/perl-lexer/src/lexer/driver.rs +++ b/crates/perl-lexer/src/lexer/driver.rs @@ -30,7 +30,6 @@ impl<'a> PerlLexer<'a> { current_quote_op: None, qw_recovery_enabled: true, eof_emitted: false, - start_time: std::time::Instant::now(), } } diff --git a/crates/perl-lexer/src/lexer/state.rs b/crates/perl-lexer/src/lexer/state.rs index 0fbdcb9682..4d22a0c85f 100644 --- a/crates/perl-lexer/src/lexer/state.rs +++ b/crates/perl-lexer/src/lexer/state.rs @@ -1,5 +1,3 @@ -use std::time::Instant; - use perl_position_tracking::Position; use crate::config::LexerConfig; @@ -35,5 +33,4 @@ pub struct PerlLexer<'a> { pub(crate) current_quote_op: Option, pub(crate) qw_recovery_enabled: bool, pub(crate) eof_emitted: bool, - pub(crate) start_time: Instant, } diff --git a/crates/perl-lexer/src/lib.rs b/crates/perl-lexer/src/lib.rs index fb13ed74cb..bef63a8242 100644 --- a/crates/perl-lexer/src/lib.rs +++ b/crates/perl-lexer/src/lib.rs @@ -174,9 +174,7 @@ use crate::lexer::helpers::{ empty_arc, is_builtin_function, is_compound_operator, is_keyword_fast, is_perl_punctuation_variable, is_quote_op_word_prefix, truncate_preview, }; -use crate::limits::{ - HEREDOC_TIMEOUT_MS, MAX_DELIM_NEST, MAX_HEREDOC_BYTES, MAX_HEREDOC_DEPTH, MAX_REGEX_BYTES, -}; +use crate::limits::{MAX_DELIM_NEST, MAX_HEREDOC_BYTES, MAX_HEREDOC_DEPTH, MAX_REGEX_BYTES}; impl<'a> PerlLexer<'a> { /// Create a new lexer that emits `HeredocBody` tokens (for LSP folding) @@ -238,18 +236,6 @@ impl<'a> PerlLexer<'a> { // Scan line by line looking for the terminator while self.position < self.input.len() { - // Timeout protection (Issue #443) - if self.start_time.elapsed().as_millis() > HEREDOC_TIMEOUT_MS as u128 { - self.pending_heredocs.remove(0); - self.position = self.input.len(); - return Some(Token { - token_type: TokenType::Error(Arc::from("Heredoc parsing timeout")), - text: Arc::from(&self.input[body_start..]), - start: body_start, - end: self.input.len(), - }); - } - // Budget cap for huge bodies - optimized check if self.position - body_start > MAX_HEREDOC_BYTES { // Remove the pending heredoc to avoid infinite loop @@ -531,7 +517,6 @@ impl<'a> PerlLexer<'a> { let saved_line_start_offset = self.line_start_offset; let saved_current_quote_op = self.current_quote_op.clone(); let saved_eof_emitted = self.eof_emitted; - let saved_start_time = self.start_time; let token = self.next_token(); @@ -551,7 +536,6 @@ impl<'a> PerlLexer<'a> { self.line_start_offset = saved_line_start_offset; self.current_quote_op = saved_current_quote_op; self.eof_emitted = saved_eof_emitted; - self.start_time = saved_start_time; token } @@ -592,7 +576,6 @@ impl<'a> PerlLexer<'a> { self.line_start_offset = 0; self.current_quote_op = None; self.eof_emitted = false; - self.start_time = std::time::Instant::now(); } /// Switch the lexer into format-body parsing mode. diff --git a/crates/perl-lexer/src/limits.rs b/crates/perl-lexer/src/limits.rs index 2e27d1d046..393442c82b 100644 --- a/crates/perl-lexer/src/limits.rs +++ b/crates/perl-lexer/src/limits.rs @@ -9,7 +9,6 @@ pub(crate) const MAX_REGEX_BYTES: usize = 64 * 1024; // 64KB max for regex patte pub(crate) const MAX_HEREDOC_BYTES: usize = 256 * 1024; // 256KB max for heredoc bodies pub(crate) const MAX_DELIM_NEST: usize = 128; // Max nesting depth for delimiters pub(crate) const MAX_HEREDOC_DEPTH: usize = 100; // Max nesting depth for heredocs -pub(crate) const HEREDOC_TIMEOUT_MS: u64 = 5000; // 5 seconds timeout for heredoc parsing /// Maximum scan iterations for a single regex literal. /// This is a lexer parse budget, not regex-engine backtracking detection. diff --git a/crates/perl-parser/src/incremental/lex.rs b/crates/perl-parser/src/incremental/lex.rs index dbe20acea0..f7880a161e 100644 --- a/crates/perl-parser/src/incremental/lex.rs +++ b/crates/perl-parser/src/incremental/lex.rs @@ -194,9 +194,7 @@ mod tests { let replayed = capture_live_checkpoint(source, expected.position) .ok_or_else(|| anyhow::anyhow!("live checkpoint replay failed"))?; - let mut expected_without_operation_time = expected.clone(); - expected_without_operation_time.start_time = replayed.start_time; - assert_eq!(replayed, expected_without_operation_time); + assert_eq!(replayed, *expected); Ok(()) } @@ -223,7 +221,7 @@ mod tests { ); assert!( capture_live_checkpoint(source, queued.position).is_none(), - "timeout-sensitive queued-heredoc checkpoints must fall back" + "queued-heredoc checkpoints must fall back before the deterministic heredoc budget path" ); let resumed = lexed @@ -239,4 +237,39 @@ mod tests { ); Ok(()) } + + #[test] + fn replayable_checkpoint_reaches_deterministic_heredoc_budget() -> Result<()> { + let body = "x".repeat(256 * 1024 + 1); + let source = format!("my $value = < Tes #[test] fn stateful_and_source_boundary_edits_match_fresh_lexing() -> TestResult { let fixtures = [ - ("division", "my $x = 10 / 2; my $after = 1;", "/ 2", "/ 3"), - ("regex", "my $ok = /foo/; my $after = 1;", "foo", "bar"), - ("quote-single", "my $x = q{foo}; my $after = 1;", "foo", "bar"), - ("quote-double", "my $x = qq{foo}; my $after = 1;", "foo", "bar"), - ("quote-words", "my @x = qw(foo bar); my $after = 1;", "foo", "baz"), - ("quote-command", "my $x = qx{echo foo}; my $after = 1;", "foo", "bar"), - ("substitution", "$x =~ s/foo/bar/; my $after = 1;", "foo", "baz"), - ("transliteration", "$x =~ tr/a-z/A-Z/; my $after = 1;", "a-z", "b-z"), - ("prototype", "sub f($$) { return 1; } my $after = 1;", "return 1", "return 2"), - ("unicode", "my $x = \"café\"; my $after = 1;", "é", "ø"), - ("crlf", "my $x = 1;\r\nmy $y = 2;\r\n", "= 2", "= 3"), - ("heredoc-body", "my $value = < TestResult { } #[test] -fn timeout_sensitive_checkpoint_falls_back_with_downstream_span_parity() -> TestResult { +fn queued_heredoc_checkpoint_falls_back_with_downstream_span_parity() -> TestResult { let source = "my $value = < Date: Wed, 12 Aug 2026 02:36:24 -0400 Subject: [PATCH 48/48] fix(parser): keep timeout repair merge-compatible --- crates/perl-lexer/src/checkpoint/core.rs | 12 ++++++------ crates/perl-lexer/src/lexer/driver.rs | 1 + crates/perl-lexer/src/lexer/state.rs | 3 +++ crates/perl-lexer/src/lib.rs | 19 ++++++++++++++++++- crates/perl-lexer/src/limits.rs | 1 + 5 files changed, 29 insertions(+), 7 deletions(-) diff --git a/crates/perl-lexer/src/checkpoint/core.rs b/crates/perl-lexer/src/checkpoint/core.rs index f780aadd08..31730262c0 100644 --- a/crates/perl-lexer/src/checkpoint/core.rs +++ b/crates/perl-lexer/src/checkpoint/core.rs @@ -25,9 +25,9 @@ pub struct QuoteOperatorCheckpoint { /// A checkpoint that captures all mutable lexer state needed for token replay. /// -/// Input references are deliberately not persisted. Heredoc recovery uses the -/// deterministic byte budget configured by the lexer, and checkpoints that -/// already have queued heredocs are rejected by the incremental replay layer. +/// Input references are deliberately not persisted. Checkpoints that already +/// have queued heredocs are rejected by the incremental replay layer before +/// re-entering the timeout-sensitive heredoc path. #[derive(Debug, Clone, PartialEq)] pub struct LexerCheckpoint { /// Current position in the input. @@ -152,8 +152,8 @@ impl LexerCheckpoint { } /// Whether restoring this checkpoint would re-enter a pending-heredoc - /// byte-budget path. Incremental callers fail closed for this state so - /// heredoc recovery remains owned by a fresh full lex. + /// timeout-sensitive path. Incremental callers fail closed for this state + /// so heredoc recovery remains owned by a fresh full lex. #[must_use] pub fn is_timeout_sensitive(&self) -> bool { !self.pending_heredocs.is_empty() @@ -356,7 +356,7 @@ pub trait Checkpointable { /// Restore mutable replay state into a lexer for the target input. /// - /// The target lexer retains its configured deterministic recovery policy. + /// The target lexer retains its configured recovery policy. fn restore(&mut self, checkpoint: &LexerCheckpoint); /// Check whether every source-relative checkpoint offset is valid. diff --git a/crates/perl-lexer/src/lexer/driver.rs b/crates/perl-lexer/src/lexer/driver.rs index df455b8f89..f7d2c0bc4d 100644 --- a/crates/perl-lexer/src/lexer/driver.rs +++ b/crates/perl-lexer/src/lexer/driver.rs @@ -30,6 +30,7 @@ impl<'a> PerlLexer<'a> { current_quote_op: None, qw_recovery_enabled: true, eof_emitted: false, + start_time: std::time::Instant::now(), } } diff --git a/crates/perl-lexer/src/lexer/state.rs b/crates/perl-lexer/src/lexer/state.rs index 4d22a0c85f..0fbdcb9682 100644 --- a/crates/perl-lexer/src/lexer/state.rs +++ b/crates/perl-lexer/src/lexer/state.rs @@ -1,3 +1,5 @@ +use std::time::Instant; + use perl_position_tracking::Position; use crate::config::LexerConfig; @@ -33,4 +35,5 @@ pub struct PerlLexer<'a> { pub(crate) current_quote_op: Option, pub(crate) qw_recovery_enabled: bool, pub(crate) eof_emitted: bool, + pub(crate) start_time: Instant, } diff --git a/crates/perl-lexer/src/lib.rs b/crates/perl-lexer/src/lib.rs index bef63a8242..fb13ed74cb 100644 --- a/crates/perl-lexer/src/lib.rs +++ b/crates/perl-lexer/src/lib.rs @@ -174,7 +174,9 @@ use crate::lexer::helpers::{ empty_arc, is_builtin_function, is_compound_operator, is_keyword_fast, is_perl_punctuation_variable, is_quote_op_word_prefix, truncate_preview, }; -use crate::limits::{MAX_DELIM_NEST, MAX_HEREDOC_BYTES, MAX_HEREDOC_DEPTH, MAX_REGEX_BYTES}; +use crate::limits::{ + HEREDOC_TIMEOUT_MS, MAX_DELIM_NEST, MAX_HEREDOC_BYTES, MAX_HEREDOC_DEPTH, MAX_REGEX_BYTES, +}; impl<'a> PerlLexer<'a> { /// Create a new lexer that emits `HeredocBody` tokens (for LSP folding) @@ -236,6 +238,18 @@ impl<'a> PerlLexer<'a> { // Scan line by line looking for the terminator while self.position < self.input.len() { + // Timeout protection (Issue #443) + if self.start_time.elapsed().as_millis() > HEREDOC_TIMEOUT_MS as u128 { + self.pending_heredocs.remove(0); + self.position = self.input.len(); + return Some(Token { + token_type: TokenType::Error(Arc::from("Heredoc parsing timeout")), + text: Arc::from(&self.input[body_start..]), + start: body_start, + end: self.input.len(), + }); + } + // Budget cap for huge bodies - optimized check if self.position - body_start > MAX_HEREDOC_BYTES { // Remove the pending heredoc to avoid infinite loop @@ -517,6 +531,7 @@ impl<'a> PerlLexer<'a> { let saved_line_start_offset = self.line_start_offset; let saved_current_quote_op = self.current_quote_op.clone(); let saved_eof_emitted = self.eof_emitted; + let saved_start_time = self.start_time; let token = self.next_token(); @@ -536,6 +551,7 @@ impl<'a> PerlLexer<'a> { self.line_start_offset = saved_line_start_offset; self.current_quote_op = saved_current_quote_op; self.eof_emitted = saved_eof_emitted; + self.start_time = saved_start_time; token } @@ -576,6 +592,7 @@ impl<'a> PerlLexer<'a> { self.line_start_offset = 0; self.current_quote_op = None; self.eof_emitted = false; + self.start_time = std::time::Instant::now(); } /// Switch the lexer into format-body parsing mode. diff --git a/crates/perl-lexer/src/limits.rs b/crates/perl-lexer/src/limits.rs index 393442c82b..2e27d1d046 100644 --- a/crates/perl-lexer/src/limits.rs +++ b/crates/perl-lexer/src/limits.rs @@ -9,6 +9,7 @@ pub(crate) const MAX_REGEX_BYTES: usize = 64 * 1024; // 64KB max for regex patte pub(crate) const MAX_HEREDOC_BYTES: usize = 256 * 1024; // 256KB max for heredoc bodies pub(crate) const MAX_DELIM_NEST: usize = 128; // Max nesting depth for delimiters pub(crate) const MAX_HEREDOC_DEPTH: usize = 100; // Max nesting depth for heredocs +pub(crate) const HEREDOC_TIMEOUT_MS: u64 = 5000; // 5 seconds timeout for heredoc parsing /// Maximum scan iterations for a single regex literal. /// This is a lexer parse budget, not regex-engine backtracking detection.