feat(parser): persist generation-bound lexer restart checkpoints (#6982)

crates/perl-lexer/src/checkpoint/mod.rs

@@ -5,7 +5,10 @@ mod core;
 mod diff;
 
 pub use cache::CheckpointCache;
-pub use core::{CheckpointContext, Checkpointable, LexerCheckpoint};
+pub use core::{
+    CheckpointContext, Checkpointable, LexerCheckpoint, PendingHeredocCheckpoint,
+    QuoteOperatorCheckpoint,
+};
 pub use diff::CheckpointDiff;
 
 #[cfg(test)]

crates/perl-lexer/src/checkpoint_impl.rs

@@ -1,22 +1,25 @@
-use crate::checkpoint::Checkpointable;
+use crate::checkpoint::{Checkpointable, PendingHeredocCheckpoint, QuoteOperatorCheckpoint};
+use crate::heredoc::HeredocSpec;
+use crate::quote_handler::QuoteOperatorInfo;
 use crate::{LexerCheckpoint, LexerMode, PerlLexer, checkpoint};
+use std::sync::Arc;
 
 impl Checkpointable for PerlLexer<'_> {
     fn checkpoint(&self) -> LexerCheckpoint {
         use checkpoint::CheckpointContext;
 
-        // Determine the checkpoint context based on current state
         let context = if matches!(self.mode, LexerMode::InFormatBody) {
             CheckpointContext::Format {
                 // Format bodies are consumed atomically by `next_token`, so a
-                // checkpoint can observe this mode only at the exact position
-                // where format-body parsing will begin.
+                // checkpoint can observe this mode only where body parsing begins.
                 start_position: self.position,
             }
         } else if !self.delimiter_stack.is_empty() {
-            // We're in some kind of quote-like construct
             CheckpointContext::QuoteLike {
-                operator: String::new(), // Would need to track this
+                operator: self
+                    .current_quote_op
+                    .as_ref()
+                    .map_or_else(String::new, |quote| quote.operator.clone()),
                 delimiter: self.delimiter_stack.last().copied().unwrap_or('\0'),
                 is_paired: true,
             }
@@ -36,6 +39,24 @@ impl Checkpointable for PerlLexer<'_> {
             after_var_subscript: self.after_var_subscript,
             paren_depth: self.paren_depth,
             current_pos: self.current_pos,
+            after_newline: self.after_newline,
+            pending_heredocs: self
+                .pending_heredocs
+                .iter()
+                .map(|pending| PendingHeredocCheckpoint {
+                    label: pending.label.to_string(),
+                    body_start: pending.body_start,
+                    allow_indent: pending.allow_indent,
+                })
+                .collect(),
+            line_start_offset: self.line_start_offset,
+            emit_heredoc_body_tokens: self.emit_heredoc_body_tokens,
+            current_quote_op: self.current_quote_op.as_ref().map(|quote| QuoteOperatorCheckpoint {
+                operator: quote.operator.clone(),
+                delimiter: quote.delimiter,
+                start_pos: quote.start_pos,
+            }),
+            qw_recovery_enabled: self.qw_recovery_enabled,
             eof_emitted: self.eof_emitted,
             context,
         }
@@ -53,27 +74,42 @@ impl Checkpointable for PerlLexer<'_> {
         self.after_var_subscript = checkpoint.after_var_subscript;
         self.paren_depth = checkpoint.paren_depth;
         self.current_pos = checkpoint.current_pos;
+        self.after_newline = checkpoint.after_newline;
+        self.pending_heredocs = checkpoint
+            .pending_heredocs
+            .iter()
+            .map(|pending| HeredocSpec {
+                label: Arc::from(pending.label.as_str()),
+                body_start: pending.body_start,
+                allow_indent: pending.allow_indent,
+            })
+            .collect();
+        self.line_start_offset = checkpoint.line_start_offset;
+        self.emit_heredoc_body_tokens = checkpoint.emit_heredoc_body_tokens;
+        self.current_quote_op =
+            checkpoint.current_quote_op.as_ref().map(|quote| QuoteOperatorInfo {
+                operator: quote.operator.clone(),
+                delimiter: quote.delimiter,
+                start_pos: quote.start_pos,
+            });
+        self.qw_recovery_enabled = checkpoint.qw_recovery_enabled;
         self.eof_emitted = checkpoint.eof_emitted;
 
-        // Handle special contexts
         use checkpoint::CheckpointContext;
-        if let CheckpointContext::Format { .. } = &checkpoint.context {
-            // Ensure we're in format body mode
-            if !matches!(self.mode, LexerMode::InFormatBody) {
-                self.mode = LexerMode::InFormatBody;
-            }
+        if matches!(checkpoint.context, CheckpointContext::Format { .. }) {
+            self.mode = LexerMode::InFormatBody;
         }
     }
 
     fn can_restore(&self, checkpoint: &LexerCheckpoint) -> bool {
-        // Can restore if the position is valid for our input
-        checkpoint.position <= self.input.len()
+        checkpoint.is_valid_for(self.input)
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::Position;
     use crate::checkpoint::CheckpointContext;
 
     type TestResult = std::result::Result<(), String>;
@@ -145,4 +181,36 @@ mod tests {
         }
         Ok(())
     }
+
+    #[test]
+    fn restore_round_trip_preserves_every_mutable_replay_field() {
+        let input = "x".repeat(96);
+        let mut lexer = PerlLexer::new(&input);
+        lexer.position = 32;
+        lexer.mode = LexerMode::ExpectOperator;
+        lexer.delimiter_stack = vec!['{', '('];
+        lexer.in_prototype = true;
+        lexer.prototype_depth = 2;
+        lexer.after_sub = true;
+        lexer.after_arrow = true;
+        lexer.hash_brace_depth = 3;
+        lexer.after_var_subscript = true;
+        lexer.paren_depth = 4;
+        lexer.current_pos = Position { byte: 32, line: 3, column: 5 };
+        lexer.after_newline = false;
+        lexer.pending_heredocs =
+            vec![HeredocSpec { label: Arc::from("END"), body_start: 48, allow_indent: true }];
+        lexer.line_start_offset = 24;
+        lexer.emit_heredoc_body_tokens = true;
+        lexer.current_quote_op =
+            Some(QuoteOperatorInfo { operator: "s".to_string(), delimiter: '{', start_pos: 28 });
+        lexer.qw_recovery_enabled = false;
+        lexer.eof_emitted = false;
+
+        let expected = lexer.checkpoint();
+        let mut restored = PerlLexer::new(&input);
+        restored.restore(&expected);
+
+        assert_eq!(restored.checkpoint(), expected);
+    }
 }

crates/perl-lexer/tests/checkpoint_edit_validity.rs

@@ -0,0 +1,47 @@
+use perl_lexer::checkpoint::{Checkpointable, LexerCheckpoint, QuoteOperatorCheckpoint};
+use perl_lexer::{PerlLexer, Position};
+
+#[test]
+fn shifted_replay_position_fails_closed_without_target_source_coordinates() {
+    let mut checkpoint = LexerCheckpoint::at_position(8);
+    checkpoint.current_pos = Position::new(8, 2, 3);
+
+    let preserved = checkpoint.try_apply_edit(0, 0, 3);
+
+    assert!(!preserved, "byte lengths cannot reconstruct shifted line and column state");
+    assert_eq!(checkpoint.position, 11, "compatibility inspection keeps the shifted byte");
+    assert_ne!(checkpoint.current_pos.byte, checkpoint.position);
+
+    let lexer = PerlLexer::new("xxxmy $value = 1;\n");
+    assert!(!checkpoint.is_valid_for("xxxmy $value = 1;\n"));
+    assert!(!lexer.can_restore(&checkpoint));
+}
+
+#[test]
+fn compatibility_edit_cannot_turn_overlapped_quote_state_into_a_restorable_default() {
+    let mut checkpoint = LexerCheckpoint::at_position(12);
+    checkpoint.current_quote_op = Some(QuoteOperatorCheckpoint {
+        operator: "qq".to_string(),
+        delimiter: '{',
+        start_pos: 9,
+    });
+
+    checkpoint.apply_edit(8, 5, 1);
+
+    assert_eq!(checkpoint.position, 8, "compatibility inspection retains the edit boundary");
+    assert!(!checkpoint.is_valid_for("xxxxxxxxrest"));
+
+    let lexer = PerlLexer::new("xxxxxxxxrest");
+    assert!(!lexer.can_restore(&checkpoint));
+}
+
+#[test]
+fn exact_boundary_edit_keeps_a_live_checkpoint_replayable() {
+    let mut checkpoint = LexerCheckpoint::at_position(8);
+    checkpoint.current_pos = Position::new(8, 1, 9);
+
+    assert!(checkpoint.try_apply_edit(8, 2, 5));
+    assert_eq!(checkpoint.position, 8);
+    assert_eq!(checkpoint.current_pos, Position::new(8, 1, 9));
+    assert!(checkpoint.is_valid_for("12345678replacement"));
+}

crates/perl-parser/src/incremental/diagnostics.rs

@@ -1,13 +1,66 @@
 use lsp_types::Diagnostic;
+use perl_parser_core::error::ParseOutput;
 use std::ops::Range;
 
-/// Result of incremental reparse
+/// Lexer work strategy selected for one incremental parse result.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum LexRestartStrategy {
+    /// Reuse the current token stream without performing lexer work.
+    Unchanged,
+    /// Lex the complete current source from byte zero.
+    FullRelex,
+    /// Restore a complete checkpoint reproduced by replaying the old prefix.
+    #[deprecated(note = "Use StoredCheckpointToEof; replay is no longer the canonical path.")]
+    LiveCheckpointToEof,
+    /// Restore a complete generation-bound checkpoint without replaying old bytes.
+    StoredCheckpointToEof,
+}
+
+/// Truthful lexer restart and token-retention receipt.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[non_exhaustive]
+pub struct LexRestartReport {
+    /// Strategy that produced the current token stream.
+    pub strategy: LexRestartStrategy,
+    /// Byte boundary where fresh lexing began.
+    pub restart_byte: usize,
+    /// Number of old-source prefix bytes replayed only to reconstruct state.
+    pub old_prefix_bytes_replayed: usize,
+    /// Number of current-source bytes lexed from restart to EOF.
+    pub relexed_bytes: usize,
+    /// Tokens before the restart boundary retained without re-lexing.
+    pub reused_prefix_tokens: usize,
+    /// Tokens after a synchronization boundary retained from the old suffix.
+    pub reused_suffix_tokens: usize,
+    /// Complete generation-bound checkpoints retained in the resulting state.
+    pub stored_checkpoint_count: usize,
+}
+
+impl LexRestartReport {
+    /// Total old tokens retained by the selected strategy.
+    #[must_use]
+    pub fn reused_tokens(self) -> usize {
+        self.reused_prefix_tokens.saturating_add(self.reused_suffix_tokens)
+    }
+}
+
+/// Result of incremental reparse.
 #[derive(Debug)]
 #[non_exhaustive]
 pub struct ReparseResult {
+    /// Byte ranges reparsed or replaced by the selected strategy.
     pub changed_ranges: Vec<Range<usize>>,
+    /// Authoritative native parser output for the current source generation.
+    pub parse_output: ParseOutput,
+    /// Legacy LSP-shaped diagnostics retained for compatibility.
     pub diagnostics: Vec<Diagnostic>,
+    /// Lexer restart, fresh-work, and token-retention receipt.
+    pub lex_restart: LexRestartReport,
+    /// Number of source bytes covered by parser reparsing work.
     pub reparsed_bytes: usize,
+    /// Compatibility total of old lexer tokens retained from prefix and suffix.
     pub reused_tokens: usize,
+    /// Total token count in the resulting incremental state.
     pub token_count: usize,
 }