diff --git a/crates/perl-lexer/src/checkpoint/core.rs b/crates/perl-lexer/src/checkpoint/core.rs index 12285efddd..31730262c0 100644 --- a/crates/perl-lexer/src/checkpoint/core.rs +++ b/crates/perl-lexer/src/checkpoint/core.rs @@ -1,76 +1,113 @@ use crate::{LexerMode, Position}; use std::fmt; -/// A checkpoint that captures the complete lexer state +/// Replay-safe representation of one queued heredoc. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PendingHeredocCheckpoint { + /// Heredoc terminator label. + pub label: String, + /// Byte offset where the heredoc body begins. + pub body_start: usize, + /// Whether `<<~` indentation is allowed. + pub allow_indent: bool, +} + +/// Replay-safe representation of an in-progress quote-like operator. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct QuoteOperatorCheckpoint { + /// Operator name such as `q`, `qr`, `s`, or `tr`. + pub operator: String, + /// Opening delimiter. + pub delimiter: char, + /// Byte offset where the operator begins. + pub start_pos: usize, +} + +/// A checkpoint that captures all mutable lexer state needed for token replay. +/// +/// Input references are deliberately not persisted. Checkpoints that already +/// have queued heredocs are rejected by the incremental replay layer before +/// re-entering the timeout-sensitive heredoc path. #[derive(Debug, Clone, PartialEq)] pub struct LexerCheckpoint { - /// Current position in the input + /// Current position in the input. pub position: usize, - /// Current lexer mode (`ExpectTerm`, `ExpectOperator`, etc.) + /// Current lexer mode (`ExpectTerm`, `ExpectOperator`, etc.). pub mode: LexerMode, - /// Stack for nested delimiters in s{}{} constructs + /// Stack for nested delimiters in `s{}{} ` constructs. pub delimiter_stack: Vec, - /// Whether we're inside prototype parens after 'sub' + /// Whether the lexer is inside prototype parentheses after `sub`. pub in_prototype: bool, - /// Paren depth to track when we exit prototype + /// Parenthesis depth used to detect the end of a prototype. pub prototype_depth: usize, - /// Whether we just saw 'sub' and are waiting for a possible prototype + /// Whether `sub` was just emitted and a prototype may follow. pub after_sub: bool, - /// Whether we just saw '->' (suppresses s/tr/y as substitution) + /// Whether `->` was just emitted, suppressing quote-op interpretation. pub after_arrow: bool, /// Depth of hash-subscript brace nesting. - /// When > 0, suppresses quote-op detection inside hash subscripts/slices. pub hash_brace_depth: usize, - /// Whether the lexer just emitted a complete $var/@var/%var token. - /// Used by the `{` handler to distinguish hash subscript openers from block openers. + /// Whether the lexer just emitted a complete variable token. pub after_var_subscript: bool, - /// Depth of open parentheses (used to guard heredoc vs bitshift disambiguation) + /// Depth of open parentheses used by heredoc/bitshift disambiguation. pub paren_depth: usize, - /// Current position with line/column tracking + /// Current position with line/column tracking. pub current_pos: Position, + /// Whether the previous consumed source unit ended a line. + pub after_newline: bool, + /// Ordered heredoc queue waiting for body consumption. + pub pending_heredocs: Vec, + /// Byte offset of the current physical line start. + pub line_start_offset: usize, + /// Whether heredoc body tokens are emitted instead of consumed virtually. + pub emit_heredoc_body_tokens: bool, + /// In-progress quote-operator metadata, when present. + pub current_quote_op: Option, + /// Whether malformed `qw` constructs use the recovery path. + pub qw_recovery_enabled: bool, /// Whether the terminal EOF token has already been emitted. pub eof_emitted: bool, - /// Additional context for complex states + /// Additional context for complex states. pub context: CheckpointContext, } -/// Additional context that may be needed for certain lexer states +/// Additional context that may be needed for certain lexer states. #[derive(Debug, Clone, PartialEq)] pub enum CheckpointContext { - /// Normal lexing + /// Normal lexing. Normal, - /// Inside a heredoc (tracks the terminator) + /// Inside a heredoc. Heredoc { - /// The terminator label (e.g. `END` in `<, }, - /// Inside a quote-like operator + /// Inside a quote-like operator. QuoteLike { - /// The operator name (e.g. `q`, `qq`, `qw`) + /// Operator name such as `q`, `qq`, or `qw`. operator: String, - /// The delimiter character (e.g. `(` in `qw(...)`) + /// Opening delimiter. delimiter: char, - /// Whether the delimiter is a paired bracket (e.g. `(` / `)`) + /// Whether the delimiter is paired. is_paired: bool, }, } impl LexerCheckpoint { - /// Create a new checkpoint with default values + /// Create a checkpoint with the default lexer state. + #[must_use] pub fn new() -> Self { Self { position: 0, @@ -84,22 +121,46 @@ impl LexerCheckpoint { after_var_subscript: false, paren_depth: 0, current_pos: Position::start(), + after_newline: true, + pending_heredocs: Vec::new(), + line_start_offset: 0, + emit_heredoc_body_tokens: false, + current_quote_op: None, + qw_recovery_enabled: true, eof_emitted: false, context: CheckpointContext::Normal, } } - /// Create a checkpoint at a specific position + /// Create a default-state checkpoint at a specific byte position. + /// + /// Line and column remain the default summary values; the byte component is + /// aligned so the checkpoint remains structurally valid for compatibility + /// callers that only model a byte boundary. + #[must_use] pub fn at_position(position: usize) -> Self { - Self { position, ..Self::new() } + let mut checkpoint = Self::new(); + checkpoint.position = position; + checkpoint.current_pos.byte = position; + checkpoint } - /// Check if this checkpoint is at the start of input + /// Check whether this checkpoint is at the start of input. + #[must_use] pub fn is_at_start(&self) -> bool { self.position == 0 } - /// Calculate the difference between two checkpoints + /// Whether restoring this checkpoint would re-enter a pending-heredoc + /// timeout-sensitive path. Incremental callers fail closed for this state + /// so heredoc recovery remains owned by a fresh full lex. + #[must_use] + pub fn is_timeout_sensitive(&self) -> bool { + !self.pending_heredocs.is_empty() + } + + /// Calculate the difference between two checkpoints. + #[must_use] pub fn diff(&self, other: &Self) -> super::CheckpointDiff { super::CheckpointDiff { position_delta: self.position as isize - other.position as isize, @@ -113,57 +174,158 @@ impl LexerCheckpoint { || self.after_var_subscript != other.after_var_subscript || self.paren_depth != other.paren_depth, eof_state_changed: self.eof_emitted != other.eof_emitted, - context_changed: self.context != other.context, + context_changed: self.context != other.context + || self.after_newline != other.after_newline + || self.pending_heredocs != other.pending_heredocs + || self.line_start_offset != other.line_start_offset + || self.emit_heredoc_body_tokens != other.emit_heredoc_body_tokens + || self.current_quote_op != other.current_quote_op + || self.qw_recovery_enabled != other.qw_recovery_enabled, } } - /// Apply an edit to this checkpoint. - /// - /// # Behavior - /// - /// * Edit before the checkpoint and ending strictly before it: the byte - /// `position` is shifted by `new_len - old_len`. The `current_pos` - /// line/column tracker is reset to `Position::start()` because we - /// cannot recompute line/column without rescanning the input. - /// * Edit overlapping the checkpoint: the checkpoint is invalidated -- - /// `position` is rewound to `start`, lexer mode and stacks are reset to - /// defaults, and `current_pos` is reset to `Position::start()`. - /// * Edit at or after the checkpoint: no change. + /// Apply an edit to source-relative checkpoint offsets. /// - /// `current_pos` is intentionally reset in both the "shifted" and - /// "invalidated" branches so callers always observe a known sentinel value - /// and must rescan from `position` to recover accurate line/column data. + /// This compatibility method retains transformed offsets for inspection, + /// but any edit that cannot preserve complete replay state leaves an + /// explicitly unrestorable checkpoint. Call [`Self::try_apply_edit`] when + /// the caller must branch on that result. pub fn apply_edit(&mut self, start: usize, old_len: usize, new_len: usize) { - if self.position > start { - if self.position >= start.saturating_add(old_len) { - self.position = self.position.saturating_sub(old_len).saturating_add(new_len); - if let CheckpointContext::Format { start_position } = &mut self.context { - *start_position = self.position; - } - self.current_pos = Position::start(); - self.eof_emitted = false; - } else { - self.position = start; - self.current_pos = Position::start(); - self.eof_emitted = false; - self.mode = LexerMode::ExpectTerm; - self.delimiter_stack.clear(); - self.in_prototype = false; - self.prototype_depth = 0; - self.after_sub = false; - self.after_arrow = false; - self.hash_brace_depth = 0; - self.after_var_subscript = false; - self.paren_depth = 0; - self.context = CheckpointContext::Normal; + let _ = self.try_apply_edit(start, old_len, new_len); + } + + /// Apply an edit and report whether all required replay state survived. + /// + /// An edit overlapping a required state offset invalidates the checkpoint. + /// A shift of the replay position also fails closed because byte counts do + /// not contain enough information to recompute line and column. Offsets are + /// still transformed for compatibility inspection, but [`Self::is_valid_for`] + /// rejects the result. An edit beginning exactly at an offset leaves it + /// anchored before the replacement so the new text is re-lexed. + #[must_use] + pub fn try_apply_edit(&mut self, start: usize, old_len: usize, new_len: usize) -> bool { + let original_position = self.position; + let Some(position) = transform_offset(self.position, start, old_len, new_len) else { + self.invalidate_at(start); + return false; + }; + let Some(line_start_offset) = + transform_offset(self.line_start_offset, start, old_len, new_len) + else { + self.invalidate_at(start); + return false; + }; + + let mut pending_heredocs = self.pending_heredocs.clone(); + for pending in &mut pending_heredocs { + let Some(body_start) = transform_offset(pending.body_start, start, old_len, new_len) + else { + self.invalidate_at(start); + return false; + }; + pending.body_start = body_start; + } + + let mut current_quote_op = self.current_quote_op.clone(); + if let Some(quote) = &mut current_quote_op { + let Some(start_pos) = transform_offset(quote.start_pos, start, old_len, new_len) else { + self.invalidate_at(start); + return false; + }; + quote.start_pos = start_pos; + } + + let mut context = self.context.clone(); + let context_valid = match &mut context { + CheckpointContext::Format { start_position } => { + transform_offset(*start_position, start, old_len, new_len) + .map(|shifted| *start_position = shifted) + .is_some() + } + CheckpointContext::Regex { flags_position, .. } => { + flags_position.as_mut().is_none_or(|flags| { + transform_offset(*flags, start, old_len, new_len) + .map(|shifted| *flags = shifted) + .is_some() + }) } + CheckpointContext::Normal + | CheckpointContext::Heredoc { .. } + | CheckpointContext::QuoteLike { .. } => true, + }; + if !context_valid { + self.invalidate_at(start); + return false; } + + self.position = position; + self.line_start_offset = line_start_offset; + self.pending_heredocs = pending_heredocs; + self.current_quote_op = current_quote_op; + self.context = context; + self.eof_emitted = false; + if self.position != original_position { + self.mark_unrestorable(); + return false; + } + true } - /// Validate that this checkpoint is valid for the given input + /// Validate all source-relative checkpoint offsets and replay identity for an input. + #[must_use] pub fn is_valid_for(&self, input: &str) -> bool { - self.position <= input.len() + self.current_pos.byte != usize::MAX + && offset_is_valid(input, self.current_pos.byte) + && offset_is_valid(input, self.position) + && offset_is_valid(input, self.line_start_offset) + && self.line_start_offset <= self.position + && self + .pending_heredocs + .iter() + .all(|pending| offset_is_valid(input, pending.body_start)) + && self.current_quote_op.as_ref().is_none_or(|quote| { + offset_is_valid(input, quote.start_pos) && quote.start_pos <= self.position + }) + && match &self.context { + CheckpointContext::Format { start_position } => { + offset_is_valid(input, *start_position) + } + CheckpointContext::Regex { flags_position, .. } => { + flags_position.is_none_or(|position| offset_is_valid(input, position)) + } + CheckpointContext::Normal + | CheckpointContext::Heredoc { .. } + | CheckpointContext::QuoteLike { .. } => true, + } + } + + fn invalidate_at(&mut self, start: usize) { + let mut reset = Self::at_position(start); + reset.mark_unrestorable(); + *self = reset; } + + fn mark_unrestorable(&mut self) { + self.current_pos = Position::start(); + // Keep the invalidation marker distinct from a live checkpoint whose + // legacy line/column summary has not advanced with ordinary lexing. + self.current_pos.byte = usize::MAX; + } +} + +fn transform_offset(offset: usize, start: usize, old_len: usize, new_len: usize) -> Option { + let old_end = start.saturating_add(old_len); + if offset <= start { + Some(offset) + } else if offset >= old_end { + Some(offset.saturating_sub(old_len).saturating_add(new_len)) + } else { + None + } +} + +fn offset_is_valid(input: &str, offset: usize) -> bool { + offset <= input.len() && input.is_char_boundary(offset) } impl Default for LexerCheckpoint { @@ -173,27 +335,30 @@ impl Default for LexerCheckpoint { } impl fmt::Display for LexerCheckpoint { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { write!( - f, - "Checkpoint@{} mode={:?} delims={} proto={} after_sub={}", + formatter, + "Checkpoint@{} mode={:?} delims={} proto={} after_sub={} heredocs={}", self.position, self.mode, self.delimiter_stack.len(), self.in_prototype, - self.after_sub + self.after_sub, + self.pending_heredocs.len() ) } } -/// Trait for types that support checkpointing +/// Trait for lexers that support state checkpointing. pub trait Checkpointable { - /// Create a checkpoint of the current state + /// Capture all mutable state required to replay tokenization. fn checkpoint(&self) -> LexerCheckpoint; - /// Restore state from a checkpoint + /// Restore mutable replay state into a lexer for the target input. + /// + /// The target lexer retains its configured recovery policy. fn restore(&mut self, checkpoint: &LexerCheckpoint); - /// Check if we can restore to a given checkpoint + /// Check whether every source-relative checkpoint offset is valid. fn can_restore(&self, checkpoint: &LexerCheckpoint) -> bool; } diff --git a/crates/perl-lexer/src/checkpoint/mod.rs b/crates/perl-lexer/src/checkpoint/mod.rs index e1cfd45092..720a6a05bb 100644 --- a/crates/perl-lexer/src/checkpoint/mod.rs +++ b/crates/perl-lexer/src/checkpoint/mod.rs @@ -5,7 +5,10 @@ mod core; mod diff; pub use cache::CheckpointCache; -pub use core::{CheckpointContext, Checkpointable, LexerCheckpoint}; +pub use core::{ + CheckpointContext, Checkpointable, LexerCheckpoint, PendingHeredocCheckpoint, + QuoteOperatorCheckpoint, +}; pub use diff::CheckpointDiff; #[cfg(test)] diff --git a/crates/perl-lexer/src/checkpoint_impl.rs b/crates/perl-lexer/src/checkpoint_impl.rs index 552979a149..15c885a8ee 100644 --- a/crates/perl-lexer/src/checkpoint_impl.rs +++ b/crates/perl-lexer/src/checkpoint_impl.rs @@ -1,22 +1,25 @@ -use crate::checkpoint::Checkpointable; +use crate::checkpoint::{Checkpointable, PendingHeredocCheckpoint, QuoteOperatorCheckpoint}; +use crate::heredoc::HeredocSpec; +use crate::quote_handler::QuoteOperatorInfo; use crate::{LexerCheckpoint, LexerMode, PerlLexer, checkpoint}; +use std::sync::Arc; impl Checkpointable for PerlLexer<'_> { fn checkpoint(&self) -> LexerCheckpoint { use checkpoint::CheckpointContext; - // Determine the checkpoint context based on current state let context = if matches!(self.mode, LexerMode::InFormatBody) { CheckpointContext::Format { // Format bodies are consumed atomically by `next_token`, so a - // checkpoint can observe this mode only at the exact position - // where format-body parsing will begin. + // checkpoint can observe this mode only where body parsing begins. start_position: self.position, } } else if !self.delimiter_stack.is_empty() { - // We're in some kind of quote-like construct CheckpointContext::QuoteLike { - operator: String::new(), // Would need to track this + operator: self + .current_quote_op + .as_ref() + .map_or_else(String::new, |quote| quote.operator.clone()), delimiter: self.delimiter_stack.last().copied().unwrap_or('\0'), is_paired: true, } @@ -36,6 +39,24 @@ impl Checkpointable for PerlLexer<'_> { after_var_subscript: self.after_var_subscript, paren_depth: self.paren_depth, current_pos: self.current_pos, + after_newline: self.after_newline, + pending_heredocs: self + .pending_heredocs + .iter() + .map(|pending| PendingHeredocCheckpoint { + label: pending.label.to_string(), + body_start: pending.body_start, + allow_indent: pending.allow_indent, + }) + .collect(), + line_start_offset: self.line_start_offset, + emit_heredoc_body_tokens: self.emit_heredoc_body_tokens, + current_quote_op: self.current_quote_op.as_ref().map(|quote| QuoteOperatorCheckpoint { + operator: quote.operator.clone(), + delimiter: quote.delimiter, + start_pos: quote.start_pos, + }), + qw_recovery_enabled: self.qw_recovery_enabled, eof_emitted: self.eof_emitted, context, } @@ -53,27 +74,42 @@ impl Checkpointable for PerlLexer<'_> { self.after_var_subscript = checkpoint.after_var_subscript; self.paren_depth = checkpoint.paren_depth; self.current_pos = checkpoint.current_pos; + self.after_newline = checkpoint.after_newline; + self.pending_heredocs = checkpoint + .pending_heredocs + .iter() + .map(|pending| HeredocSpec { + label: Arc::from(pending.label.as_str()), + body_start: pending.body_start, + allow_indent: pending.allow_indent, + }) + .collect(); + self.line_start_offset = checkpoint.line_start_offset; + self.emit_heredoc_body_tokens = checkpoint.emit_heredoc_body_tokens; + self.current_quote_op = + checkpoint.current_quote_op.as_ref().map(|quote| QuoteOperatorInfo { + operator: quote.operator.clone(), + delimiter: quote.delimiter, + start_pos: quote.start_pos, + }); + self.qw_recovery_enabled = checkpoint.qw_recovery_enabled; self.eof_emitted = checkpoint.eof_emitted; - // Handle special contexts use checkpoint::CheckpointContext; - if let CheckpointContext::Format { .. } = &checkpoint.context { - // Ensure we're in format body mode - if !matches!(self.mode, LexerMode::InFormatBody) { - self.mode = LexerMode::InFormatBody; - } + if matches!(checkpoint.context, CheckpointContext::Format { .. }) { + self.mode = LexerMode::InFormatBody; } } fn can_restore(&self, checkpoint: &LexerCheckpoint) -> bool { - // Can restore if the position is valid for our input - checkpoint.position <= self.input.len() + checkpoint.is_valid_for(self.input) } } #[cfg(test)] mod tests { use super::*; + use crate::Position; use crate::checkpoint::CheckpointContext; type TestResult = std::result::Result<(), String>; @@ -145,4 +181,36 @@ mod tests { } Ok(()) } + + #[test] + fn restore_round_trip_preserves_every_mutable_replay_field() { + let input = "x".repeat(96); + let mut lexer = PerlLexer::new(&input); + lexer.position = 32; + lexer.mode = LexerMode::ExpectOperator; + lexer.delimiter_stack = vec!['{', '(']; + lexer.in_prototype = true; + lexer.prototype_depth = 2; + lexer.after_sub = true; + lexer.after_arrow = true; + lexer.hash_brace_depth = 3; + lexer.after_var_subscript = true; + lexer.paren_depth = 4; + lexer.current_pos = Position { byte: 32, line: 3, column: 5 }; + lexer.after_newline = false; + lexer.pending_heredocs = + vec![HeredocSpec { label: Arc::from("END"), body_start: 48, allow_indent: true }]; + lexer.line_start_offset = 24; + lexer.emit_heredoc_body_tokens = true; + lexer.current_quote_op = + Some(QuoteOperatorInfo { operator: "s".to_string(), delimiter: '{', start_pos: 28 }); + lexer.qw_recovery_enabled = false; + lexer.eof_emitted = false; + + let expected = lexer.checkpoint(); + let mut restored = PerlLexer::new(&input); + restored.restore(&expected); + + assert_eq!(restored.checkpoint(), expected); + } } diff --git a/crates/perl-lexer/tests/checkpoint_edit_validity.rs b/crates/perl-lexer/tests/checkpoint_edit_validity.rs new file mode 100644 index 0000000000..d46b3045ae --- /dev/null +++ b/crates/perl-lexer/tests/checkpoint_edit_validity.rs @@ -0,0 +1,47 @@ +use perl_lexer::checkpoint::{Checkpointable, LexerCheckpoint, QuoteOperatorCheckpoint}; +use perl_lexer::{PerlLexer, Position}; + +#[test] +fn shifted_replay_position_fails_closed_without_target_source_coordinates() { + let mut checkpoint = LexerCheckpoint::at_position(8); + checkpoint.current_pos = Position::new(8, 2, 3); + + let preserved = checkpoint.try_apply_edit(0, 0, 3); + + assert!(!preserved, "byte lengths cannot reconstruct shifted line and column state"); + assert_eq!(checkpoint.position, 11, "compatibility inspection keeps the shifted byte"); + assert_ne!(checkpoint.current_pos.byte, checkpoint.position); + + let lexer = PerlLexer::new("xxxmy $value = 1;\n"); + assert!(!checkpoint.is_valid_for("xxxmy $value = 1;\n")); + assert!(!lexer.can_restore(&checkpoint)); +} + +#[test] +fn compatibility_edit_cannot_turn_overlapped_quote_state_into_a_restorable_default() { + let mut checkpoint = LexerCheckpoint::at_position(12); + checkpoint.current_quote_op = Some(QuoteOperatorCheckpoint { + operator: "qq".to_string(), + delimiter: '{', + start_pos: 9, + }); + + checkpoint.apply_edit(8, 5, 1); + + assert_eq!(checkpoint.position, 8, "compatibility inspection retains the edit boundary"); + assert!(!checkpoint.is_valid_for("xxxxxxxxrest")); + + let lexer = PerlLexer::new("xxxxxxxxrest"); + assert!(!lexer.can_restore(&checkpoint)); +} + +#[test] +fn exact_boundary_edit_keeps_a_live_checkpoint_replayable() { + let mut checkpoint = LexerCheckpoint::at_position(8); + checkpoint.current_pos = Position::new(8, 1, 9); + + assert!(checkpoint.try_apply_edit(8, 2, 5)); + assert_eq!(checkpoint.position, 8); + assert_eq!(checkpoint.current_pos, Position::new(8, 1, 9)); + assert!(checkpoint.is_valid_for("12345678replacement")); +} diff --git a/crates/perl-parser/src/incremental/diagnostics.rs b/crates/perl-parser/src/incremental/diagnostics.rs index 5729baf62c..5366488d49 100644 --- a/crates/perl-parser/src/incremental/diagnostics.rs +++ b/crates/perl-parser/src/incremental/diagnostics.rs @@ -1,13 +1,66 @@ use lsp_types::Diagnostic; +use perl_parser_core::error::ParseOutput; use std::ops::Range; -/// Result of incremental reparse +/// Lexer work strategy selected for one incremental parse result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum LexRestartStrategy { + /// Reuse the current token stream without performing lexer work. + Unchanged, + /// Lex the complete current source from byte zero. + FullRelex, + /// Restore a complete checkpoint reproduced by replaying the old prefix. + #[deprecated(note = "Use StoredCheckpointToEof; replay is no longer the canonical path.")] + LiveCheckpointToEof, + /// Restore a complete generation-bound checkpoint without replaying old bytes. + StoredCheckpointToEof, +} + +/// Truthful lexer restart and token-retention receipt. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct LexRestartReport { + /// Strategy that produced the current token stream. + pub strategy: LexRestartStrategy, + /// Byte boundary where fresh lexing began. + pub restart_byte: usize, + /// Number of old-source prefix bytes replayed only to reconstruct state. + pub old_prefix_bytes_replayed: usize, + /// Number of current-source bytes lexed from restart to EOF. + pub relexed_bytes: usize, + /// Tokens before the restart boundary retained without re-lexing. + pub reused_prefix_tokens: usize, + /// Tokens after a synchronization boundary retained from the old suffix. + pub reused_suffix_tokens: usize, + /// Complete generation-bound checkpoints retained in the resulting state. + pub stored_checkpoint_count: usize, +} + +impl LexRestartReport { + /// Total old tokens retained by the selected strategy. + #[must_use] + pub fn reused_tokens(self) -> usize { + self.reused_prefix_tokens.saturating_add(self.reused_suffix_tokens) + } +} + +/// Result of incremental reparse. #[derive(Debug)] #[non_exhaustive] pub struct ReparseResult { + /// Byte ranges reparsed or replaced by the selected strategy. pub changed_ranges: Vec>, + /// Authoritative native parser output for the current source generation. + pub parse_output: ParseOutput, + /// Legacy LSP-shaped diagnostics retained for compatibility. pub diagnostics: Vec, + /// Lexer restart, fresh-work, and token-retention receipt. + pub lex_restart: LexRestartReport, + /// Number of source bytes covered by parser reparsing work. pub reparsed_bytes: usize, + /// Compatibility total of old lexer tokens retained from prefix and suffix. pub reused_tokens: usize, + /// Total token count in the resulting incremental state. pub token_count: usize, } diff --git a/crates/perl-parser/src/incremental/lex.rs b/crates/perl-parser/src/incremental/lex.rs index 213bdf6c43..6bf0a9794e 100644 --- a/crates/perl-parser/src/incremental/lex.rs +++ b/crates/perl-parser/src/incremental/lex.rs @@ -1,42 +1,455 @@ use crate::incremental::LineIndex; use crate::incremental::checkpoint::LexCheckpoint; -use perl_lexer::{LexerMode, Token, TokenType}; +use crate::incremental::edit::Edit; +use anyhow::Result; +use perl_lexer::{ + Checkpointable, LexerCheckpoint as LiveLexerCheckpoint, PerlLexer, Token, TokenType, +}; -pub(crate) fn create_lex_checkpoints( - tokens: &[Token], +/// Hard cap for complete behavior-bearing checkpoints retained in one state. +pub const MAX_STORED_LEX_CHECKPOINTS: usize = 4096; +const INITIAL_CHECKPOINT_SPACING: usize = 128; + +/// One complete checkpoint bound to the exact source generation that produced it. +#[derive(Clone)] +pub(crate) struct StoredLexCheckpoint { + pub(crate) summary: LexCheckpoint, + pub(crate) live: LiveLexerCheckpoint, + source_fingerprint: u64, + prefix_fingerprint: u64, +} + +impl StoredLexCheckpoint { + fn capture(source: &str, live: LiveLexerCheckpoint, line_index: &LineIndex) -> Self { + let summary = summarize_checkpoint(&live, line_index); + let prefix = source.get(..live.position).unwrap_or_default(); + Self { + summary, + live, + source_fingerprint: fingerprint(source.as_bytes()), + prefix_fingerprint: fingerprint(prefix.as_bytes()), + } + } + + fn belongs_to_source(&self, source: &str) -> bool { + if self.live.position > source.len() || !source.is_char_boundary(self.live.position) { + return false; + } + let Some(prefix) = source.get(..self.live.position) else { + return false; + }; + self.source_fingerprint == fingerprint(source.as_bytes()) + && self.prefix_fingerprint == fingerprint(prefix.as_bytes()) + && self.live.is_valid_for(source) + } + + /// Clone and transform the complete state for one old-generation edit. + pub(crate) fn prepare_for_edit( + &self, + old_source: &str, + edit: &Edit, + ) -> Option { + if !self.belongs_to_source(old_source) + || self.summary.byte > edit.start_byte + || self.live.is_timeout_sensitive() + { + return None; + } + let old_len = edit.old_end_byte.checked_sub(edit.start_byte)?; + let mut live = self.live.clone(); + live.try_apply_edit(edit.start_byte, old_len, edit.new_text.len()).then_some(live) + } + + /// Carry an old prefix checkpoint into the edited generation when all + /// behavior-bearing offsets and prefix bytes remain valid. + pub(crate) fn transform_for_generation( + &self, + old_source: &str, + new_source: &str, + edit: &Edit, + ) -> Option { + if !self.belongs_to_source(old_source) || self.summary.byte >= edit.start_byte { + return None; + } + let old_len = edit.old_end_byte.checked_sub(edit.start_byte)?; + let mut live = self.live.clone(); + if !live.try_apply_edit(edit.start_byte, old_len, edit.new_text.len()) + || !live.is_valid_for(new_source) + { + return None; + } + let prefix = new_source.get(..live.position)?; + if fingerprint(prefix.as_bytes()) != self.prefix_fingerprint { + return None; + } + Some(Self { + summary: self.summary, + live, + source_fingerprint: fingerprint(new_source.as_bytes()), + prefix_fingerprint: self.prefix_fingerprint, + }) + } +} + +pub(crate) struct LexedSource { + pub(crate) tokens: Vec, + pub(crate) checkpoints: Vec, + pub(crate) stored_checkpoints: Vec, + #[cfg(test)] + pub(crate) live_checkpoints: Vec, +} + +fn fingerprint(bytes: &[u8]) -> u64 { + let mut value = 0xcbf2_9ce4_8422_2325_u64; + for byte in bytes { + value ^= u64::from(*byte); + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + value +} + +fn summarize_checkpoint(checkpoint: &LiveLexerCheckpoint, line_index: &LineIndex) -> LexCheckpoint { + let (line, column) = line_index.byte_to_position(checkpoint.position); + LexCheckpoint { byte: checkpoint.position, mode: checkpoint.mode, line, column } +} + +fn push_summary( + summaries: &mut Vec, + checkpoint: &LiveLexerCheckpoint, + line_index: &LineIndex, +) { + if summaries.last().is_none_or(|summary| summary.byte != checkpoint.position) { + summaries.push(summarize_checkpoint(checkpoint, line_index)); + } +} + +fn behavior_state_changed(previous: &LiveLexerCheckpoint, current: &LiveLexerCheckpoint) -> bool { + previous.mode != current.mode + || previous.delimiter_stack != current.delimiter_stack + || previous.in_prototype != current.in_prototype + || previous.prototype_depth != current.prototype_depth + || previous.after_sub != current.after_sub + || previous.after_arrow != current.after_arrow + || previous.hash_brace_depth != current.hash_brace_depth + || previous.after_var_subscript != current.after_var_subscript + || previous.paren_depth != current.paren_depth + || previous.after_newline != current.after_newline + || previous.pending_heredocs != current.pending_heredocs + || previous.line_start_offset != current.line_start_offset + || previous.emit_heredoc_body_tokens != current.emit_heredoc_body_tokens + || previous.current_quote_op != current.current_quote_op + || previous.qw_recovery_enabled != current.qw_recovery_enabled + || previous.eof_emitted != current.eof_emitted + || previous.context != current.context +} + +fn compact_stored_checkpoints(checkpoints: &mut Vec) { + let original = std::mem::take(checkpoints); + let original_len = original.len(); + checkpoints.extend(original.into_iter().enumerate().filter_map(|(index, checkpoint)| { + (index == 0 || index % 2 == 0 || index + 1 == original_len).then_some(checkpoint) + })); +} + +fn push_stored_checkpoint( + source: &str, + line_index: &LineIndex, + checkpoints: &mut Vec, + spacing: &mut usize, + live: &LiveLexerCheckpoint, +) { + if checkpoints.last().is_some_and(|stored| stored.live.position == live.position) { + return; + } + + let retain = checkpoints.last().is_none_or(|previous| { + live.position.saturating_sub(previous.live.position) >= *spacing + || behavior_state_changed(&previous.live, live) + }); + if !retain { + return; + } + + if checkpoints.len() >= MAX_STORED_LEX_CHECKPOINTS { + compact_stored_checkpoints(checkpoints); + *spacing = spacing.saturating_mul(2).max(1); + } + checkpoints.push(StoredLexCheckpoint::capture(source, live.clone(), line_index)); +} + +/// Lex one complete source and capture complete, generation-bound restart state. +pub(crate) fn lex_source_with_checkpoints(source: &str, line_index: &LineIndex) -> LexedSource { + let mut lexer = PerlLexer::new(source); + let mut tokens = Vec::new(); + let mut checkpoints = Vec::new(); + let mut stored_checkpoints = Vec::new(); + let mut checkpoint_spacing = INITIAL_CHECKPOINT_SPACING; + #[cfg(test)] + let mut live_checkpoints = Vec::new(); + + loop { + let live = lexer.checkpoint(); + push_summary(&mut checkpoints, &live, line_index); + push_stored_checkpoint( + source, + line_index, + &mut stored_checkpoints, + &mut checkpoint_spacing, + &live, + ); + #[cfg(test)] + live_checkpoints.push(live); + + let Some(token) = lexer.next_token() else { + break; + }; + if token.token_type == TokenType::EOF { + let terminal = lexer.checkpoint(); + push_summary(&mut checkpoints, &terminal, line_index); + push_stored_checkpoint( + source, + line_index, + &mut stored_checkpoints, + &mut checkpoint_spacing, + &terminal, + ); + break; + } + tokens.push(token); + } + + LexedSource { + tokens, + checkpoints, + stored_checkpoints, + #[cfg(test)] + live_checkpoints, + } +} + +/// Restore complete mutable state into the edited source and re-lex to EOF. +pub(crate) fn lex_from_live_checkpoint( + source: &str, line_index: &LineIndex, -) -> Vec { - let mut checkpoints = - vec![LexCheckpoint { byte: 0, mode: LexerMode::ExpectTerm, line: 0, column: 0 }]; - let mut mode = LexerMode::ExpectTerm; - for token in tokens { - mode = match token.token_type { - TokenType::Semicolon | TokenType::LeftBrace | TokenType::RightBrace => { - let (line, column) = line_index.byte_to_position(token.end); - checkpoints.push(LexCheckpoint { - byte: token.end, - mode: LexerMode::ExpectTerm, - line, - column, - }); - LexerMode::ExpectTerm - } - TokenType::Keyword(ref kw) if kw.as_ref() == "sub" || kw.as_ref() == "package" => { - let (line, column) = line_index.byte_to_position(token.start); - checkpoints.push(LexCheckpoint { - byte: token.start, - mode: LexerMode::ExpectTerm, - line, - column, - }); - LexerMode::ExpectTerm - } - TokenType::Identifier(_) | TokenType::Number(_) | TokenType::StringLiteral => { - LexerMode::ExpectOperator - } - TokenType::Operator(_) => LexerMode::ExpectTerm, - _ => mode, - } - } - checkpoints + checkpoint: &LiveLexerCheckpoint, +) -> Result { + let mut lexer = PerlLexer::new(source); + if !lexer.can_restore(checkpoint) { + anyhow::bail!("stored lexer checkpoint is not valid for the edited source"); + } + lexer.restore(checkpoint); + + let mut tokens = Vec::new(); + let mut checkpoints = Vec::new(); + let mut stored_checkpoints = Vec::new(); + let mut checkpoint_spacing = INITIAL_CHECKPOINT_SPACING; + #[cfg(test)] + let mut live_checkpoints = Vec::new(); + let mut last_position = checkpoint.position; + + loop { + let live = lexer.checkpoint(); + push_summary(&mut checkpoints, &live, line_index); + push_stored_checkpoint( + source, + line_index, + &mut stored_checkpoints, + &mut checkpoint_spacing, + &live, + ); + #[cfg(test)] + live_checkpoints.push(live); + + let Some(token) = lexer.next_token() else { + break; + }; + if token.token_type == TokenType::EOF { + let terminal = lexer.checkpoint(); + push_summary(&mut checkpoints, &terminal, line_index); + push_stored_checkpoint( + source, + line_index, + &mut stored_checkpoints, + &mut checkpoint_spacing, + &terminal, + ); + break; + } + if token.end <= last_position { + anyhow::bail!("incremental lexer did not advance at byte {}", token.start); + } + last_position = token.end; + tokens.push(token); + } + + Ok(LexedSource { + tokens, + checkpoints, + stored_checkpoints, + #[cfg(test)] + live_checkpoints, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stored_checkpoint_rejects_a_different_source_generation() -> Result<()> { + let source = "my $value = 1;"; + let line_index = LineIndex::new(source); + let lexed = lex_source_with_checkpoints(source, &line_index); + let stored = lexed + .stored_checkpoints + .first() + .ok_or_else(|| anyhow::anyhow!("origin checkpoint is missing"))?; + let edit = Edit { + start_byte: source.len(), + old_end_byte: source.len(), + new_end_byte: source.len(), + new_text: String::new(), + }; + + assert!(stored.prepare_for_edit("my $value = 2;", &edit).is_none()); + Ok(()) + } + + #[test] + fn stored_checkpoint_set_is_bounded_and_retains_a_late_boundary() -> Result<()> { + let source = + (0..20_000).map(|index| format!("my $v{index} = {index};\n")).collect::(); + let line_index = LineIndex::new(&source); + let lexed = lex_source_with_checkpoints(&source, &line_index); + + assert!(!lexed.stored_checkpoints.is_empty()); + assert!(lexed.stored_checkpoints.len() <= MAX_STORED_LEX_CHECKPOINTS); + let last = lexed + .stored_checkpoints + .last() + .ok_or_else(|| anyhow::anyhow!("last checkpoint is missing"))?; + assert!(last.summary.byte > source.len() / 2); + Ok(()) + } + + #[test] + fn live_checkpoint_preserves_after_arrow_state() -> Result<()> { + let source = "$object->method();"; + let line_index = LineIndex::new(source); + let lexed = lex_source_with_checkpoints(source, &line_index); + let method_index = lexed + .tokens + .iter() + .position(|token| token.text.as_ref() == "method") + .ok_or_else(|| anyhow::anyhow!("method token is missing"))?; + let checkpoint = &lexed.live_checkpoints[method_index]; + + assert_eq!(checkpoint.position, lexed.tokens[method_index].start); + assert!(checkpoint.after_arrow, "method restart must preserve after_arrow"); + Ok(()) + } + + #[test] + fn live_checkpoint_preserves_prototype_and_nesting_state() { + let source = "sub f($$) { return 1; }"; + let line_index = LineIndex::new(source); + let lexed = lex_source_with_checkpoints(source, &line_index); + + assert!(lexed.live_checkpoints.iter().any(|checkpoint| checkpoint.in_prototype)); + assert!(lexed.live_checkpoints.iter().any(|checkpoint| checkpoint.paren_depth > 0)); + } + + #[test] + fn heredoc_queue_is_captured_but_timeout_sensitive_state_is_not_selected() -> Result<()> { + let source = "my $value = < Result<()> + { + let source = "my $value = < queued.position && checkpoint.pending_heredocs.is_empty() + }) + .ok_or_else(|| anyhow::anyhow!("checkpoint after heredoc completion is missing"))?; + assert!( + lexed.checkpoints.iter().any(|summary| summary.byte == resumed.position), + "restart summaries must resume after the heredoc queue drains" + ); + Ok(()) + } + + #[test] + fn stored_checkpoint_replay_reaches_the_deterministic_heredoc_budget() -> Result<()> { + let body = "x".repeat(256 * 1024 + 1); + let source = format!("my $value = < Result { + let mut by_start = edits.iter().collect::>(); + by_start.sort_by_key(|edit| (edit.start_byte, edit.old_end_byte)); + + let mut previous: Option<&Edit> = None; + let mut total_changed = 0usize; + + for edit in by_start { + if edit.start_byte > edit.old_end_byte || edit.old_end_byte > source.len() { + anyhow::bail!( + "incremental edit range {}..{} is invalid for source length {}", + edit.start_byte, + edit.old_end_byte, + source.len() + ); + } + if !source.is_char_boundary(edit.start_byte) || !source.is_char_boundary(edit.old_end_byte) + { + anyhow::bail!( + "incremental edit range {}..{} is not on UTF-8 boundaries", + edit.start_byte, + edit.old_end_byte + ); + } + + let expected_new_end = edit + .start_byte + .checked_add(edit.new_text.len()) + .ok_or_else(|| anyhow::anyhow!("incremental edit new range overflows usize"))?; + if edit.new_end_byte != expected_new_end { + anyhow::bail!( + "incremental edit new_end_byte {} does not match replacement end {}", + edit.new_end_byte, + expected_new_end + ); + } + + if let Some(previous) = previous { + if edit.start_byte < previous.old_end_byte || edit.start_byte == previous.start_byte { + anyhow::bail!( + "incremental edit ranges are overlapping or share an ambiguous start: {}..{} and {}..{}", + previous.start_byte, + previous.old_end_byte, + edit.start_byte, + edit.old_end_byte + ); + } + } + previous = Some(edit); + + total_changed = total_changed + .checked_add(edit.touched_bytes()) + .ok_or_else(|| anyhow::anyhow!("incremental edit byte total overflows usize"))?; + } + + Ok(total_changed) +} + +fn unchanged_result(state: &IncrementalState) -> ReparseResult { + let lex_restart = LexRestartReport { + strategy: LexRestartStrategy::Unchanged, + restart_byte: state.source().len(), + old_prefix_bytes_replayed: 0, + relexed_bytes: 0, + reused_prefix_tokens: state.tokens().len(), + reused_suffix_tokens: 0, + stored_checkpoint_count: state.stored_lex_checkpoint_count(), + }; + ReparseResult { + changed_ranges: Vec::new(), + parse_output: state.parse_output().clone(), + diagnostics: Vec::new(), + lex_restart, + reparsed_bytes: 0, + reused_tokens: lex_restart.reused_tokens(), + token_count: state.tokens().len(), + } +} + +fn apply_text_edits(state: &mut IncrementalState, edits_descending: &[Edit]) -> Result<()> { + for edit in edits_descending { + apply_text_edit_to_state(state, edit)?; + } + Ok(()) +} + +fn full_reparse_after_edits( + state: &mut IncrementalState, + edits_descending: &[Edit], +) -> Result { + let mut candidate = state.clone(); + apply_text_edits(&mut candidate, edits_descending)?; + let result = full_reparse(&mut candidate)?; + *state = candidate; + Ok(result) +} + +/// Apply edits incrementally. pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result { + let total_changed = validate_edits(state.source(), edits)?; + if edits.is_empty() { + return Ok(unchanged_result(state)); + } + let mut sorted_edits = edits.to_vec(); - sorted_edits.sort_by_key(|e| e.start_byte); + sorted_edits.sort_by_key(|edit| edit.start_byte); sorted_edits.reverse(); - let total_changed = sorted_edits.iter().map(Edit::touched_bytes).sum::(); - if total_changed > MAX_EDIT_SIZE { - return full_reparse(state); + return full_reparse_after_edits(state, &sorted_edits); } if sorted_edits.len() == 1 { let edit = &sorted_edits[0]; if edit.touched_bytes() > 1024 || edit.new_text.matches('\n').count() > 10 { - apply_text_edit_to_state(state, edit)?; - return full_reparse(state); + return full_reparse_after_edits(state, &sorted_edits); } - let reparse = match apply_single_edit(state, edit) { + let mut candidate = state.clone(); + let reparse = match apply_single_edit(&mut candidate, edit) { Ok(reparse) => reparse, - Err(_) => return full_reparse(state), + Err(_) => return full_reparse_after_edits(state, &sorted_edits), }; - let reparsed_bytes = reparse.range.end - reparse.range.start; - - // Re-parse the AST from the updated source so that state.ast reflects - // the edit (#5036). apply_single_edit only re-lexes tokens; without - // this write-back, any consumer reading state.ast after apply_edits - // gets the pre-edit tree. - reparse_ast(state); - Ok(ReparseResult { + candidate.refresh_parse_output(); + let reused_tokens = reparse.lex_restart.reused_tokens(); + let result = ReparseResult { changed_ranges: vec![reparse.range], + parse_output: candidate.parse_output().clone(), diagnostics: vec![], - reparsed_bytes, - reused_tokens: reparse.reused_tokens, + lex_restart: reparse.lex_restart, + reparsed_bytes: candidate.source().len(), + reused_tokens, token_count: reparse.token_count, - }) + }; + *state = candidate; + Ok(result) } else { - for edit in sorted_edits { - if apply_single_edit(state, &edit).is_err() { - return full_reparse(state); - } - } - full_reparse(state) + full_reparse_after_edits(state, &sorted_edits) } } -/// Re-parse the AST from the current source text without re-lexing. -/// -/// This is the AST write-back that `apply_single_edit` was missing (#5036). -/// After `apply_single_edit` updates `state.source` and `state.tokens`, this -/// function re-parses the full source to produce a fresh AST, so consumers -/// reading `state.ast` after `apply_edits` get the post-edit tree. -#[expect( - deprecated, - reason = "AST write-back is the legacy field's supported refresh boundary (#5036)" -)] -fn reparse_ast(state: &mut IncrementalState) { - let mut parser = Parser::new(&state.source); - state.ast = match parser.parse() { - Ok(ast) => ast, - Err(e) => Node::new( - NodeKind::Error { - message: e.to_string(), - expected: vec![], - found: None, - partial: None, - }, - SourceLocation { start: 0, end: state.source.len() }, - ), - }; - state.parse_checkpoints = IncrementalState::create_parse_checkpoints(&state.ast); -} - #[cfg(test)] mod tests; diff --git a/crates/perl-parser/src/incremental/reparse.rs b/crates/perl-parser/src/incremental/reparse.rs index 1314a58bbc..e4ba63b114 100644 --- a/crates/perl-parser/src/incremental/reparse.rs +++ b/crates/perl-parser/src/incremental/reparse.rs @@ -1,44 +1,31 @@ use crate::incremental::{ - IncrementalState, diagnostics::ReparseResult, edit::Edit, lex::create_lex_checkpoints, + IncrementalState, + diagnostics::{LexRestartReport, LexRestartStrategy, ReparseResult}, + edit::Edit, + lex::{lex_from_live_checkpoint, lex_source_with_checkpoints}, }; use anyhow::Result; -use perl_lexer::{PerlLexer, TokenType}; -use perl_parser_core::ast::{Node, NodeKind, SourceLocation}; -use perl_parser_core::parser::Parser; -use ropey::Rope; use std::ops::Range; pub(crate) struct SingleEditReparse { pub(crate) range: Range, - pub(crate) reused_tokens: usize, + pub(crate) lex_restart: LexRestartReport, pub(crate) token_count: usize, } -/// Apply a (possibly negative) `byte_shift` to a token offset without wrapping. -/// -/// A negative `byte_shift` whose magnitude exceeds `offset` would underflow when -/// `(offset as isize + byte_shift)` is cast back to `usize`, silently wrapping to -/// a huge value and corrupting the token stream. Clamp the result to 0 instead so -/// the shifted offset stays valid (fixes #2471). -fn shift_offset(offset: usize, byte_shift: isize) -> usize { - (offset as isize).saturating_add(byte_shift).max(0) as usize -} - pub(crate) fn apply_text_edit_to_state(state: &mut IncrementalState, edit: &Edit) -> Result<()> { - let old_end = edit.old_end_byte.min(state.source.len()); - let start = edit.start_byte.min(state.source.len()); - if !state.source.is_char_boundary(start) || !state.source.is_char_boundary(old_end) { + let old_end = edit.old_end_byte.min(state.source().len()); + let start = edit.start_byte.min(state.source().len()); + if !state.source().is_char_boundary(start) || !state.source().is_char_boundary(old_end) { anyhow::bail!("edit range is not on UTF-8 boundaries"); } let mut new_source = - String::with_capacity(state.source.len() - (old_end - start) + edit.new_text.len()); - new_source.push_str(&state.source[..start]); + String::with_capacity(state.source().len() - (old_end - start) + edit.new_text.len()); + new_source.push_str(&state.source()[..start]); new_source.push_str(&edit.new_text); - new_source.push_str(&state.source[old_end..]); - state.source = new_source; - state.rope = Rope::from_str(&state.source); - state.line_index = perl_line_index::LineIndex::new(&state.source); + new_source.push_str(&state.source()[old_end..]); + state.replace_source_text(new_source); Ok(()) } @@ -47,172 +34,209 @@ pub(crate) fn apply_single_edit( state: &mut IncrementalState, edit: &Edit, ) -> Result { - let Some(cp) = state.find_lex_checkpoint(edit.start_byte).copied() else { - apply_text_edit_to_state(state, edit)?; - anyhow::bail!("No checkpoint found"); - }; - let old_end = edit.old_end_byte.min(state.source.len()); - let start = edit.start_byte.min(state.source.len()); - let byte_shift = edit.new_text.len() as isize - (old_end - start) as isize; + let old_source = state.source().to_string(); + let selected = state + .stored_lex_checkpoints() + .iter() + .rev() + .filter(|stored| stored.summary.byte <= edit.start_byte) + .find_map(|stored| { + stored.prepare_for_edit(&old_source, edit).map(|live| (stored.summary, live)) + }) + .ok_or_else(|| anyhow::anyhow!("No valid stored lexer checkpoint found"))?; + let (summary, live_checkpoint) = selected; + let restart_byte = live_checkpoint.position; + let reused_prefix_tokens = + state.tokens().iter().take_while(|token| token.start < restart_byte).count(); + let old_prefix_checkpoints = state + .stored_lex_checkpoints() + .iter() + .take_while(|checkpoint| checkpoint.summary.byte < restart_byte) + .cloned() + .collect::>(); + apply_text_edit_to_state(state, edit)?; + let lexed = lex_from_live_checkpoint(state.source(), state.line_index(), &live_checkpoint)?; - use perl_lexer::{Checkpointable, LexerCheckpoint, Position}; - let mut lexer = PerlLexer::new(&state.source); - let mut lex_cp = LexerCheckpoint::new(); - lex_cp.position = cp.byte; - lex_cp.mode = cp.mode; - lex_cp.current_pos = - Position { byte: cp.byte, line: (cp.line + 1) as u32, column: (cp.column + 1) as u32 }; - lexer.restore(&lex_cp); - let start_idx = - state.tokens.iter().position(|t| t.start >= cp.byte).unwrap_or(state.tokens.len()); - let edit_end_in_new = start + edit.new_text.len(); - let old_sync_start = - state.tokens.iter().position(|t| t.start >= old_end).unwrap_or(state.tokens.len()); - let mut new_tokens = Vec::new(); - let mut last = cp.byte; - let mut synced = false; - let mut sync_old_idx = state.tokens.len(); - while let Some(token) = lexer.next_token() { - if token.token_type == TokenType::EOF { - break; - } - if token.end <= last { - anyhow::bail!("incremental lexer did not advance at byte {}", token.start); - } - last = token.end; - if token.start >= edit_end_in_new { - let mut found = false; - for (off, old_tok) in state.tokens[old_sync_start..].iter().enumerate() { - let shifted_start = shift_offset(old_tok.start, byte_shift); - let shifted_end = shift_offset(old_tok.end, byte_shift); - if token.start == shifted_start - && token.end == shifted_end - && token.token_type == old_tok.token_type - { - found = true; - sync_old_idx = old_sync_start + off + 1; - break; - } - } - new_tokens.push(token); - if found { - synced = true; - break; - } - } else { - new_tokens.push(token); - } - } - let reused_tokens = if synced { state.tokens.len().saturating_sub(sync_old_idx) } else { 0 }; - if synced { - for old_tok in &state.tokens[sync_old_idx..] { - let mut adjusted = old_tok.clone(); - adjusted.start = shift_offset(adjusted.start, byte_shift); - adjusted.end = shift_offset(adjusted.end, byte_shift); - last = adjusted.end; - new_tokens.push(adjusted); - } - } - state.tokens.splice(start_idx.., new_tokens); - state.lex_checkpoints = create_lex_checkpoints(&state.tokens, &state.line_index); - Ok(SingleEditReparse { range: cp.byte..last, reused_tokens, token_count: state.tokens.len() }) + let mut tokens = state.tokens()[..reused_prefix_tokens].to_vec(); + tokens.extend(lexed.tokens); + + let mut checkpoint_summaries = state + .lex_checkpoints() + .iter() + .take_while(|checkpoint| checkpoint.byte < restart_byte) + .copied() + .collect::>(); + checkpoint_summaries.extend(lexed.checkpoints); + + let mut stored_checkpoints = old_prefix_checkpoints + .iter() + .filter_map(|checkpoint| { + checkpoint.transform_for_generation(&old_source, state.source(), edit) + }) + .collect::>(); + stored_checkpoints.extend(lexed.stored_checkpoints); + + state.replace_lex_state(tokens, checkpoint_summaries, stored_checkpoints); + + let lex_restart = LexRestartReport { + strategy: LexRestartStrategy::StoredCheckpointToEof, + restart_byte, + old_prefix_bytes_replayed: 0, + relexed_bytes: state.source().len().saturating_sub(restart_byte), + reused_prefix_tokens, + reused_suffix_tokens: 0, + stored_checkpoint_count: state.stored_lex_checkpoint_count(), + }; + + debug_assert_eq!(summary.byte, restart_byte); + Ok(SingleEditReparse { + range: restart_byte..state.source().len(), + lex_restart, + token_count: state.tokens().len(), + }) } -#[expect(deprecated, reason = "full reparse is the legacy AST field's supported refresh boundary")] pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result { - let mut parser = Parser::new(&state.source); - state.ast = match parser.parse() { - Ok(ast) => ast, - Err(e) => Node::new( - NodeKind::Error { - message: e.to_string(), - expected: vec![], - found: None, - partial: None, - }, - SourceLocation { start: 0, end: state.source.len() }, - ), + state.refresh_parse_output(); + let source_len = state.source().len(); + let lexed = lex_source_with_checkpoints(state.source(), state.line_index()); + state.replace_lex_state(lexed.tokens, lexed.checkpoints, lexed.stored_checkpoints); + + let lex_restart = LexRestartReport { + strategy: LexRestartStrategy::FullRelex, + restart_byte: 0, + old_prefix_bytes_replayed: 0, + relexed_bytes: source_len, + reused_prefix_tokens: 0, + reused_suffix_tokens: 0, + stored_checkpoint_count: state.stored_lex_checkpoint_count(), }; - let mut lexer = PerlLexer::new(&state.source); - let mut tokens = Vec::new(); - while let Some(token) = lexer.next_token() { - if token.token_type == TokenType::EOF { - break; - } - tokens.push(token); - } - state.tokens = tokens; - state.rope = Rope::from_str(&state.source); - state.line_index = perl_line_index::LineIndex::new(&state.source); - state.lex_checkpoints = create_lex_checkpoints(&state.tokens, &state.line_index); - state.parse_checkpoints = IncrementalState::create_parse_checkpoints(&state.ast); + Ok(ReparseResult { - changed_ranges: vec![0..state.source.len()], + changed_ranges: vec![0..source_len], + parse_output: state.parse_output().clone(), diagnostics: vec![], - reparsed_bytes: state.source.len(), - reused_tokens: 0, - token_count: state.tokens.len(), + lex_restart, + reparsed_bytes: source_len, + reused_tokens: lex_restart.reused_tokens(), + token_count: state.tokens().len(), }) } #[cfg(test)] -mod reparse_offset_tests { +mod tests { use super::*; - use crate::incremental::IncrementalState; - use anyhow::Result; + use perl_lexer::{PerlLexer, Token, TokenType}; + + fn fresh_tokens(source: &str) -> Vec { + let mut lexer = PerlLexer::new(source); + let mut tokens = Vec::new(); + while let Some(token) = lexer.next_token() { + if token.token_type == TokenType::EOF { + break; + } + tokens.push(token); + } + tokens + } + + fn assert_tokens_equal(actual: &[Token], expected: &[Token]) { + assert_eq!(actual.len(), expected.len(), "token count diverged"); + for (index, (actual, expected)) in actual.iter().zip(expected).enumerate() { + assert_eq!(actual.token_type, expected.token_type, "token kind {index}"); + assert_eq!(actual.text, expected.text, "token payload {index}"); + assert_eq!(actual.start, expected.start, "token start {index}"); + assert_eq!(actual.end, expected.end, "token end {index}"); + } + } #[test] - fn shift_offset_clamps_negative_underflow_to_zero() { - // A negative `byte_shift` whose magnitude exceeds the offset previously - // wrapped to a huge usize via `(offset as isize + byte_shift) as usize`. - // It must clamp to 0 instead (regression for #2471). - assert_eq!(shift_offset(3, -10), 0); - assert_eq!(shift_offset(0, -1), 0); - // Exact boundary: shifting to zero is fine, one past zero clamps. - assert_eq!(shift_offset(5, -5), 0); - assert_eq!(shift_offset(5, -6), 0); - // Non-underflowing shifts are unaffected. - assert_eq!(shift_offset(5, -2), 3); - assert_eq!(shift_offset(5, 0), 5); - assert_eq!(shift_offset(5, 4), 9); + fn equal_width_edit_restores_stored_state_without_replaying_old_bytes() -> Result<()> { + let source = "my $x = 1; my $y = 2;"; + let start = source.find("= 1").ok_or_else(|| anyhow::anyhow!("literal missing"))? + 2; + let edit = Edit { + start_byte: start, + old_end_byte: start + 1, + new_end_byte: start + 1, + new_text: "9".to_string(), + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_single_edit(&mut state, &edit)?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_eq!(result.range.end, state.source().len()); + assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); + Ok(()) } #[test] - fn apply_single_edit_negative_shift_does_not_wrap_token_offsets() -> Result<()> { - // Build a document whose tokens near the start sit at small byte offsets, - // then delete a large run from the very beginning so `byte_shift` is - // negative and larger in magnitude than those token positions. Before the - // fix, the reused/sync token offsets wrapped to enormous usize values; now - // they must stay bounded by the (much smaller) new source length. - let source = "my $a = 1;\nmy $b = 2;\nmy $c = 3;\nmy $d = 4;\n".to_string(); - let mut state = IncrementalState::new(source.clone()); - - // Delete the first statement plus newline (12 bytes) from offset 0. - let delete_len = "my $a = 1;\n".len(); + fn method_name_edit_matches_fresh_lex_from_stored_state() -> Result<()> { + let source = "$object->method(); my $x = 1;"; + let start = source.find("method").ok_or_else(|| anyhow::anyhow!("method missing"))?; let edit = Edit { - start_byte: 0, - old_end_byte: delete_len, - new_end_byte: 0, - new_text: String::new(), + start_byte: start, + old_end_byte: start + "method".len(), + new_end_byte: start + "member".len(), + new_text: "member".to_string(), }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_single_edit(&mut state, &edit)?; - // `apply_single_edit` may legitimately bail (e.g. no usable checkpoint), - // in which case the wrapping branch was never reached. The invariant we - // assert is: if it succeeds, every resulting token offset is in range. - if apply_single_edit(&mut state, &edit).is_ok() { - let new_len = state.source.len(); - for tok in &state.tokens { - assert!( - tok.start <= new_len && tok.end <= new_len, - "token offset wrapped: start={}, end={}, source len={}", - tok.start, - tok.end, - new_len, - ); - assert!(tok.start <= tok.end, "token start exceeds end: {tok:?}"); - } - } + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); + assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); + Ok(()) + } + + #[test] + fn heredoc_body_edit_uses_an_earlier_safe_stored_checkpoint() -> Result<()> { + let source = "my $value = < Result<()> { + let source = "my $a = 1; my $b = 2; my $c = 3;"; + let mut state = IncrementalState::new(source.to_string()); + let first_start = + source.find("= 2").ok_or_else(|| anyhow::anyhow!("first edit missing"))? + 2; + let first = Edit { + start_byte: first_start, + old_end_byte: first_start + 1, + new_end_byte: first_start + 1, + new_text: "8".to_string(), + }; + apply_single_edit(&mut state, &first)?; + + let second_start = + state.source().find("= 3").ok_or_else(|| anyhow::anyhow!("second edit missing"))? + 2; + let second = Edit { + start_byte: second_start, + old_end_byte: second_start + 1, + new_end_byte: second_start + 1, + new_text: "9".to_string(), + }; + let result = apply_single_edit(&mut state, &second)?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); + assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); Ok(()) } } diff --git a/crates/perl-parser/src/incremental/state.rs b/crates/perl-parser/src/incremental/state.rs index acabda51e9..3f4077d12f 100644 --- a/crates/perl-parser/src/incremental/state.rs +++ b/crates/perl-parser/src/incremental/state.rs @@ -1,69 +1,181 @@ use crate::incremental::checkpoint::{LexCheckpoint, ParseCheckpoint, ScopeSnapshot}; -use crate::incremental::lex::create_lex_checkpoints; -use perl_lexer::{PerlLexer, Token, TokenType}; +use crate::incremental::lex::{StoredLexCheckpoint, lex_source_with_checkpoints}; +use perl_lexer::Token; use perl_line_index::LineIndex; -use perl_parser_core::ast::{Node, NodeKind, SourceLocation}; +use perl_parser_core::ast::{Node, NodeKind}; +use perl_parser_core::error::ParseOutput; use perl_parser_core::parser::Parser; use ropey::Rope; +use std::ops::Deref; +/// Read-only compatibility view for legacy field-style access. +#[doc(hidden)] #[derive(Clone)] -pub struct IncrementalState { - pub rope: Rope, - pub line_index: LineIndex, - pub lex_checkpoints: Vec, - pub parse_checkpoints: Vec, - /// Parsed AST. - /// - /// **Staleness invariant (#5036):** This field is only guaranteed current - /// after `new()` or `full_reparse()`. After `apply_single_edit()` it may be - /// stale relative to `tokens`/`source`. It is kept solely for - /// `create_parse_checkpoints` in `full_reparse`. Production providers - /// always use a fresh full reparse via `ParsedSnapshot`, never this field. - #[deprecated( - note = "Only valid after new()/full_reparse(); may be stale after apply_single_edit(). Use a fresh parse instead." - )] - pub ast: Node, - pub tokens: Vec, +pub struct IncrementalStateReadView { + /// Current committed source text. pub source: String, + /// Lexer restart summaries for the current committed token stream. + pub lex_checkpoints: Vec, +} + +/// One internally consistent incremental parser generation. +/// +/// Complete behavior-bearing lexer checkpoints are retained privately and move +/// atomically with source, token, line-index, and parser state. The public +/// `LexCheckpoint` vector remains a compact compatibility summary; it is not the +/// restart authority. +#[derive(Clone)] +#[non_exhaustive] +pub struct IncrementalState { + pub(super) rope: Rope, + pub(super) line_index: LineIndex, + pub(super) parse_checkpoints: Vec, + /// Authoritative native parser output for the current source. + pub(super) parse_output: ParseOutput, + /// Parsed AST compatibility mirror. + #[deprecated(note = "Use parse_output(); this compatibility mirror will be removed.")] + pub(super) ast: Node, + pub(super) tokens: Vec, + pub(super) stored_lex_checkpoints: Vec, + pub(super) read_view: IncrementalStateReadView, +} + +impl Deref for IncrementalState { + type Target = IncrementalStateReadView; + + fn deref(&self) -> &Self::Target { + &self.read_view + } } impl IncrementalState { - #[expect(deprecated, reason = "the legacy AST field seeds parse checkpoints for compatibility")] + /// Build the initial committed generation from source text. + #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] + #[must_use] pub fn new(source: String) -> Self { let rope = Rope::from_str(&source); let line_index = LineIndex::new(&source); let mut parser = Parser::new(&source); - let ast = match parser.parse() { - Ok(ast) => ast, - Err(e) => Node::new( - NodeKind::Error { - message: e.to_string(), - expected: vec![], - found: None, - partial: None, - }, - SourceLocation { start: 0, end: source.len() }, - ), - }; - let mut lexer = PerlLexer::new(&source); - let mut tokens = Vec::new(); - while let Some(token) = lexer.next_token() { - if token.token_type == TokenType::EOF { - break; - } - tokens.push(token); + let parse_output = parser.parse_with_recovery(); + let ast = parse_output.ast.clone(); + let lexed = lex_source_with_checkpoints(&source, &line_index); + let tokens = lexed.tokens; + let lex_checkpoints = lexed.checkpoints; + let stored_lex_checkpoints = lexed.stored_checkpoints; + let parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast); + Self { + rope, + line_index, + parse_checkpoints, + parse_output, + ast, + tokens, + stored_lex_checkpoints, + read_view: IncrementalStateReadView { source, lex_checkpoints }, } - let lex_checkpoints = create_lex_checkpoints(&tokens, &line_index); - let parse_checkpoints = Self::create_parse_checkpoints(&ast); - Self { rope, line_index, lex_checkpoints, parse_checkpoints, ast, tokens, source } } + + /// Current committed source text. + #[must_use] + pub fn source(&self) -> &str { + &self.read_view.source + } + + /// Rope view for the current committed source. + #[must_use] + pub fn rope(&self) -> &Rope { + &self.rope + } + + /// Line index for the current committed source. + #[must_use] + pub fn line_index(&self) -> &LineIndex { + &self.line_index + } + + /// Lexer restart summaries for the current committed token stream. + #[must_use] + pub fn lex_checkpoints(&self) -> &[LexCheckpoint] { + &self.read_view.lex_checkpoints + } + + /// Number of complete generation-bound lexer checkpoints retained privately. + #[must_use] + pub fn stored_lex_checkpoint_count(&self) -> usize { + self.stored_lex_checkpoints.len() + } + + pub(crate) fn stored_lex_checkpoints(&self) -> &[StoredLexCheckpoint] { + &self.stored_lex_checkpoints + } + + /// Parser restart summaries for the current committed parse output. + #[must_use] + pub fn parse_checkpoints(&self) -> &[ParseCheckpoint] { + &self.parse_checkpoints + } + + /// Authoritative recovery-aware parser output for this generation. + #[must_use] + pub fn parse_output(&self) -> &ParseOutput { + &self.parse_output + } + + /// Compatibility AST view for the current generation. + #[deprecated(note = "Use parse_output().ast; this compatibility view will be removed.")] + #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] + #[must_use] + pub fn ast(&self) -> &Node { + &self.ast + } + + /// Current committed lexer token stream. + #[must_use] + pub fn tokens(&self) -> &[Token] { + &self.tokens + } + + /// Find the nearest compact lexer checkpoint at or before `byte`. + #[must_use] pub fn find_lex_checkpoint(&self, byte: usize) -> Option<&LexCheckpoint> { - self.lex_checkpoints.iter().rev().find(|cp| cp.byte <= byte) + self.read_view.lex_checkpoints.iter().rev().find(|cp| cp.byte <= byte) } + + /// Find the nearest parser checkpoint at or before `byte`. + #[must_use] pub fn find_parse_checkpoint(&self, byte: usize) -> Option<&ParseCheckpoint> { self.parse_checkpoints.iter().rev().find(|cp| cp.byte <= byte) } + /// Replace the text-bearing portion of a staged generation. + pub(super) fn replace_source_text(&mut self, source: String) { + self.rope = Rope::from_str(&source); + self.line_index = LineIndex::new(&source); + self.read_view.source = source; + } + + /// Replace the staged lexer output and both checkpoint planes together. + pub(super) fn replace_lex_state( + &mut self, + tokens: Vec, + lex_checkpoints: Vec, + stored_lex_checkpoints: Vec, + ) { + self.tokens = tokens; + self.read_view.lex_checkpoints = lex_checkpoints; + self.stored_lex_checkpoints = stored_lex_checkpoints; + } + + /// Refresh the authoritative parser output from the current source. + #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] + pub(crate) fn refresh_parse_output(&mut self) { + let mut parser = Parser::new(self.source()); + let parse_output = parser.parse_with_recovery(); + self.parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast); + self.ast = parse_output.ast.clone(); + self.parse_output = parse_output; + } + pub(crate) fn create_parse_checkpoints(ast: &Node) -> Vec { let mut checkpoints = vec![]; let mut scope = ScopeSnapshot::default(); diff --git a/crates/perl-parser/src/incremental/tests.rs b/crates/perl-parser/src/incremental/tests.rs index 3b1223383c..12fd851274 100644 --- a/crates/perl-parser/src/incremental/tests.rs +++ b/crates/perl-parser/src/incremental/tests.rs @@ -1,5 +1,7 @@ use super::*; use anyhow::Result; +use perl_parser_core::ast::NodeKind; +use perl_parser_core::parser::Parser; use proptest::prelude::*; #[derive(Clone, Debug)] diff --git a/crates/perl-parser/tests/incremental_lexer_restart.rs b/crates/perl-parser/tests/incremental_lexer_restart.rs new file mode 100644 index 0000000000..13296e79c2 --- /dev/null +++ b/crates/perl-parser/tests/incremental_lexer_restart.rs @@ -0,0 +1,217 @@ +#![cfg(feature = "incremental")] +//! Public-contract tests for generation-bound stored lexer restart. + +use perl_lexer::{PerlLexer, Token, TokenType}; +use perl_parser::incremental::{LexRestartStrategy, MAX_STORED_LEX_CHECKPOINTS}; +use perl_parser::{Edit, IncrementalState, apply_edits}; + +type TestResult = Result<(), Box>; + +fn fresh_tokens(source: &str) -> Vec { + let mut lexer = PerlLexer::new(source); + let mut tokens = Vec::new(); + while let Some(token) = lexer.next_token() { + if token.token_type == TokenType::EOF { + break; + } + tokens.push(token); + } + tokens +} + +fn assert_tokens_equal(actual: &[Token], expected: &[Token]) { + assert_eq!(actual.len(), expected.len(), "token count diverged"); + for (index, (actual, expected)) in actual.iter().zip(expected).enumerate() { + assert_eq!(actual.token_type, expected.token_type, "token kind {index}"); + assert_eq!(actual.text, expected.text, "token payload {index}"); + assert_eq!(actual.start, expected.start, "token start {index}"); + assert_eq!(actual.end, expected.end, "token end {index}"); + } +} + +fn replacing_edit(source: &str, needle: &str, replacement: &str) -> Result { + let start = source + .find(needle) + .ok_or_else(|| std::io::Error::other(format!("fixture needle {needle:?} is missing")))?; + Ok(Edit { + start_byte: start, + old_end_byte: start + needle.len(), + new_end_byte: start + replacement.len(), + new_text: replacement.to_string(), + }) +} + +#[test] +fn empty_edit_batch_reports_unchanged_without_lexer_or_parser_work() -> TestResult { + let source = "my $before = 1; my $after = 2;"; + let mut state = IncrementalState::new(source.to_string()); + let token_count = state.tokens().len(); + + let result = apply_edits(&mut state, &[])?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::Unchanged); + assert_eq!(result.lex_restart.restart_byte, source.len()); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); + assert_eq!(result.lex_restart.relexed_bytes, 0); + assert_eq!(result.lex_restart.reused_prefix_tokens, token_count); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_eq!(result.reused_tokens, token_count); + assert_eq!(result.reparsed_bytes, 0); + assert!(result.lex_restart.stored_checkpoint_count > 0); + assert!(result.changed_ranges.is_empty()); + assert_eq!(state.source(), source); + assert_tokens_equal(state.tokens(), &fresh_tokens(source)); + Ok(()) +} + +#[test] +fn late_equal_width_edit_uses_stored_state_and_relexes_the_complete_suffix() -> TestResult { + let source = "my $before = 1; my $target = 2; my $after = 3;"; + let start = source.find("= 2").ok_or("target literal is missing")? + 2; + let edit = Edit { + start_byte: start, + old_end_byte: start + 1, + new_end_byte: start + 1, + new_text: "9".to_string(), + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); + assert!(result.lex_restart.restart_byte > 0); + assert!(result.lex_restart.reused_prefix_tokens > 0); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_eq!( + result.lex_restart.relexed_bytes, + state.source().len() - result.lex_restart.restart_byte + ); + assert_eq!(result.reused_tokens, result.lex_restart.reused_tokens()); + assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); + Ok(()) +} + +#[test] +fn stateful_and_source_boundary_edits_match_fresh_lexing() -> TestResult { + let fixtures = [ + ("division", "my $x = 10 / 2; my $after = 1;", "/ 2", "/ 3"), + ("regex", "my $ok = /foo/; my $after = 1;", "foo", "bar"), + ("quote-single", "my $x = q{foo}; my $after = 1;", "foo", "bar"), + ("quote-double", "my $x = qq{foo}; my $after = 1;", "foo", "bar"), + ("quote-words", "my @x = qw(foo bar); my $after = 1;", "foo", "baz"), + ("quote-command", "my $x = qx{echo foo}; my $after = 1;", "foo", "bar"), + ("substitution", "$x =~ s/foo/bar/; my $after = 1;", "foo", "baz"), + ("transliteration", "$x =~ tr/a-z/A-Z/; my $after = 1;", "a-z", "b-z"), + ("prototype", "sub f($$) { return 1; } my $after = 1;", "return 1", "return 2"), + ("unicode", "my $x = \"café\"; my $after = 1;", "é", "ø"), + ( + "crlf", + "my $x = 1; +my $y = 2; +", + "= 2", + "= 3", + ), + ( + "heredoc-body", + "my $value = < TestResult { + let source = "my $value = $object->method(); my $after = 1;"; + let start = source.find("method").ok_or("method name is missing")?; + let edit = Edit { + start_byte: start, + old_end_byte: start + "method".len(), + new_end_byte: start + "member".len(), + new_text: "member".to_string(), + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); + Ok(()) +} + +#[test] +fn large_edit_reports_full_relex_instead_of_checkpoint_reuse() -> TestResult { + let source = "my $value = 1;"; + let replacement = "my $value = 2;\n".repeat(80); + let edit = Edit { + start_byte: 0, + old_end_byte: source.len(), + new_end_byte: replacement.len(), + new_text: replacement, + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::FullRelex); + assert_eq!(result.lex_restart.restart_byte, 0); + assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); + assert_eq!(result.lex_restart.reused_prefix_tokens, 0); + assert_eq!(result.lex_restart.reused_suffix_tokens, 0); + assert_eq!(result.lex_restart.relexed_bytes, state.source().len()); + assert!(result.lex_restart.stored_checkpoint_count <= MAX_STORED_LEX_CHECKPOINTS); + assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); + Ok(()) +} + +#[test] +fn timeout_sensitive_heredoc_state_selects_an_earlier_safe_checkpoint() -> TestResult { + let source = "my $value = <= downstream_start).collect::>(); + let expected_downstream = + expected.iter().filter(|token| token.start >= downstream_start).collect::>(); + assert_eq!(actual_downstream.len(), expected_downstream.len()); + for (index, (actual, expected)) in actual_downstream.iter().zip(expected_downstream).enumerate() + { + assert_eq!(actual.token_type, expected.token_type, "downstream token kind {index}"); + assert_eq!(actual.text, expected.text, "downstream token payload {index}"); + assert_eq!(actual.start, expected.start, "downstream token start {index}"); + assert_eq!(actual.end, expected.end, "downstream token end {index}"); + } + Ok(()) +} diff --git a/crates/perl-parser/tests/incremental_parse_output.rs b/crates/perl-parser/tests/incremental_parse_output.rs new file mode 100644 index 0000000000..2faa837cef --- /dev/null +++ b/crates/perl-parser/tests/incremental_parse_output.rs @@ -0,0 +1,179 @@ +#![cfg(feature = "incremental")] +//! Differential tests for the incremental native parse-output contract. + +use perl_parser::incremental::MAX_EDIT_SIZE; +use perl_parser::{apply_edits, Edit, IncrementalState, ParseOutput, Parser}; + +type TestResult = Result<(), Box>; + +fn fresh_output(source: &str) -> ParseOutput { + let mut parser = Parser::new(source); + parser.parse_with_recovery() +} + +fn assert_output_equivalent(actual: &ParseOutput, expected: &ParseOutput) { + assert_eq!(actual.ast, expected.ast); + assert_eq!(actual.diagnostics, expected.diagnostics); + assert_eq!(actual.terminated_early, expected.terminated_early); + assert_eq!(actual.recovered_count, expected.recovered_count); + assert_eq!(actual.budget_usage.errors_emitted, expected.budget_usage.errors_emitted); + assert_eq!(actual.budget_usage.current_depth, expected.budget_usage.current_depth); + assert_eq!(actual.budget_usage.max_depth_reached, expected.budget_usage.max_depth_reached); + assert_eq!(actual.budget_usage.tokens_skipped, expected.budget_usage.tokens_skipped); + assert_eq!( + actual.budget_usage.recoveries_attempted, + expected.budget_usage.recoveries_attempted + ); +} + +fn apply_reference_edits(source: &str, edits: &[Edit]) -> String { + let mut sorted = edits.to_vec(); + sorted.sort_by_key(|edit| edit.start_byte); + sorted.reverse(); + let mut result = source.to_string(); + for edit in sorted { + result.replace_range(edit.start_byte..edit.old_end_byte, &edit.new_text); + } + result +} + +#[test] +fn initial_malformed_state_keeps_the_native_recovered_tree_and_diagnostics() { + let source = "my $x = ; print 1;"; + let state = IncrementalState::new(source.to_string()); + let fresh = fresh_output(source); + assert!(!fresh.diagnostics.is_empty()); + assert_output_equivalent(state.parse_output(), &fresh); +} + +#[test] +fn empty_edit_batch_preserves_the_current_generation_without_work() -> TestResult { + let source = "my $x = ; print 1;"; + let mut state = IncrementalState::new(source.to_string()); + let before = state.parse_output().clone(); + let token_count = state.tokens().len(); + + let result = apply_edits(&mut state, &[])?; + + assert_eq!(state.source(), source); + assert!(result.changed_ranges.is_empty()); + assert_eq!(result.reparsed_bytes, 0); + assert_eq!(result.reused_tokens, token_count); + assert_eq!(result.token_count, token_count); + assert_eq!(state.tokens().len(), token_count); + assert_output_equivalent(state.parse_output(), &before); + assert_output_equivalent(&result.parse_output, &before); + Ok(()) +} + +#[test] +fn clean_to_malformed_edit_returns_the_current_native_parse_output() -> TestResult { + let source = "my $x = 1; print 2;"; + let start = source.find("= 1").ok_or("initializer missing")? + 2; + let edit = Edit { + start_byte: start, + old_end_byte: start + 1, + new_end_byte: start, + new_text: String::new(), + }; + let final_source = apply_reference_edits(source, std::slice::from_ref(&edit)); + let fresh = fresh_output(&final_source); + assert!(!fresh.diagnostics.is_empty()); + + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(state.source(), final_source); + assert_eq!(result.reparsed_bytes, final_source.len()); + assert_output_equivalent(state.parse_output(), &fresh); + assert_output_equivalent(&result.parse_output, &fresh); + Ok(()) +} + +#[test] +fn malformed_to_clean_edit_removes_recovery_diagnostics_atomically() -> TestResult { + let source = "my $x = ; print 2;"; + let start = source.find("= ;").ok_or("insertion point missing")? + 2; + let edit = Edit { + start_byte: start, + old_end_byte: start, + new_end_byte: start + 1, + new_text: "1".to_string(), + }; + let final_source = apply_reference_edits(source, std::slice::from_ref(&edit)); + let fresh = fresh_output(&final_source); + assert!(fresh.diagnostics.is_empty()); + + let mut state = IncrementalState::new(source.to_string()); + assert!(!state.parse_output().diagnostics.is_empty()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(state.source(), final_source); + assert_output_equivalent(state.parse_output(), &fresh); + assert_output_equivalent(&result.parse_output, &fresh); + Ok(()) +} + +#[test] +fn oversized_batch_applies_every_edit_before_full_fallback() -> TestResult { + let source = "my $left = 1;\nmy $right = 2;\n"; + let second_start = source.find("my $right").ok_or("second statement missing")?; + let padding = " ".repeat(MAX_EDIT_SIZE / 2 + 1); + let edits = vec![ + Edit { + start_byte: 0, + old_end_byte: 0, + new_end_byte: padding.len(), + new_text: padding.clone(), + }, + Edit { + start_byte: second_start, + old_end_byte: second_start, + new_end_byte: second_start + padding.len(), + new_text: padding, + }, + ]; + let final_source = apply_reference_edits(source, &edits); + let fresh = fresh_output(&final_source); + + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &edits)?; + + assert_eq!(state.source(), final_source); + assert_eq!(result.changed_ranges, vec![0..final_source.len()]); + assert_eq!(result.reparsed_bytes, final_source.len()); + assert_output_equivalent(state.parse_output(), &fresh); + assert_output_equivalent(&result.parse_output, &fresh); + Ok(()) +} + +#[test] +fn invalid_overlapping_batch_leaves_the_previous_generation_untouched() -> TestResult { + let source = "my $value = 12;"; + let literal = source.find("12").ok_or("literal missing")?; + let edits = [ + Edit { + start_byte: literal, + old_end_byte: literal + 2, + new_end_byte: literal + 1, + new_text: "3".to_string(), + }, + Edit { + start_byte: literal + 1, + old_end_byte: literal + 2, + new_end_byte: literal + 2, + new_text: "4".to_string(), + }, + ]; + let before = fresh_output(source); + let mut state = IncrementalState::new(source.to_string()); + + let Err(error) = apply_edits(&mut state, &edits) else { + return Err("overlapping edit batch must fail".into()); + }; + + assert!(error.to_string().contains("overlapping")); + assert_eq!(state.source(), source); + assert_output_equivalent(state.parse_output(), &before); + Ok(()) +} diff --git a/scripts/ci/run_parser_integration.py b/scripts/ci/run_parser_integration.py index b351a46458..dc20db639f 100644 --- a/scripts/ci/run_parser_integration.py +++ b/scripts/ci/run_parser_integration.py @@ -101,7 +101,7 @@ def main() -> int: return result.returncode # Feature-gated parser tests are not exercised by the default target - # command. Keep this explicit proof in the same bounded parser gate. + # command. Keep these explicit proofs in the same bounded parser gate. incremental_command = [ "cargo", "test", @@ -112,6 +112,10 @@ def main() -> int: "incremental", "--test", "incremental_parser_accuracy", + "--test", + "incremental_parse_output", + "--test", + "incremental_lexer_restart", "--", "--test-threads=4", ]