diff --git a/crates/perl-parser/src/incremental/diagnostics.rs b/crates/perl-parser/src/incremental/diagnostics.rs index 00a3e07957..4bdb553719 100644 --- a/crates/perl-parser/src/incremental/diagnostics.rs +++ b/crates/perl-parser/src/incremental/diagnostics.rs @@ -1,3 +1,4 @@ +use crate::incremental::work::IncrementalWorkReceipt; use lsp_types::Diagnostic; use perl_parser_core::error::ParseOutput; use std::ops::Range; @@ -57,6 +58,8 @@ pub struct ReparseResult { pub diagnostics: Vec, /// Lexer restart, fresh-work, and token-retention receipt. pub lex_restart: LexRestartReport, + /// Validated production strategy and performed-work receipt. + pub work: IncrementalWorkReceipt, /// Number of source bytes covered by parser reparsing work. pub reparsed_bytes: usize, /// Compatibility total of old lexer tokens retained from prefix and suffix. diff --git a/crates/perl-parser/src/incremental/mod.rs b/crates/perl-parser/src/incremental/mod.rs index 211aa5d6da..0b2dd785d4 100644 --- a/crates/perl-parser/src/incremental/mod.rs +++ b/crates/perl-parser/src/incremental/mod.rs @@ -9,6 +9,7 @@ mod lex; mod reparse; mod state; mod strategy; +mod work; use anyhow::Result; @@ -21,6 +22,10 @@ pub use lex::MAX_STORED_LEX_CHECKPOINTS; use reparse::{apply_single_edit, apply_text_edit_to_state, full_reparse}; pub use state::IncrementalState; pub use strategy::MAX_EDIT_SIZE; +pub use work::{ + IncrementalStrategy, IncrementalWorkReceipt, IncrementalWorkReceiptError, +}; +use work::ParserInvocationReceipt; pub mod incremental_advanced_reuse; #[cfg(test)] @@ -92,7 +97,7 @@ fn validate_edits(source: &str, edits: &[Edit]) -> Result { Ok(total_changed) } -fn unchanged_result(state: &IncrementalState) -> ReparseResult { +fn unchanged_result(state: &IncrementalState) -> Result { let lex_restart = LexRestartReport { strategy: LexRestartStrategy::Unchanged, restart_byte: state.source().len(), @@ -102,15 +107,26 @@ fn unchanged_result(state: &IncrementalState) -> ReparseResult { reused_suffix_tokens: 0, stored_checkpoint_count: state.stored_lex_checkpoint_count(), }; - ReparseResult { + let work = IncrementalWorkReceipt::from_parts( + IncrementalStrategy::Unchanged, + ParserInvocationReceipt::default(), + lex_restart, + 0, + state.source().len(), + state.tokens().len(), + state.parse_output().ast.count_nodes(), + ); + work.validate()?; + Ok(ReparseResult { changed_ranges: Vec::new(), parse_output: state.parse_output().clone(), diagnostics: Vec::new(), lex_restart, + work, reparsed_bytes: 0, reused_tokens: lex_restart.reused_tokens(), token_count: state.tokens().len(), - } + }) } fn apply_text_edits(state: &mut IncrementalState, edits_descending: &[Edit]) -> Result<()> { @@ -135,7 +151,7 @@ fn full_reparse_after_edits( pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result { let total_changed = validate_edits(state.source(), edits)?; if edits.is_empty() { - return Ok(unchanged_result(state)); + return unchanged_result(state); } let mut sorted_edits = edits.to_vec(); @@ -159,13 +175,25 @@ pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result return full_reparse_after_edits(state, &sorted_edits), }; - candidate.refresh_parse_output(); + let parser_receipt = candidate.refresh_parse_output(); let reused_tokens = reparse.lex_restart.reused_tokens(); + let final_node_count = candidate.parse_output().ast.count_nodes(); + let work = IncrementalWorkReceipt::from_parts( + IncrementalStrategy::CheckpointToEofThenFullParse, + parser_receipt, + reparse.lex_restart, + reparse.fresh_tokens_emitted, + candidate.source().len(), + reparse.token_count, + final_node_count, + ); + work.validate()?; let result = ReparseResult { changed_ranges: vec![reparse.range], parse_output: candidate.parse_output().clone(), diagnostics: vec![], lex_restart: reparse.lex_restart, + work, reparsed_bytes: candidate.source().len(), reused_tokens, token_count: reparse.token_count, diff --git a/crates/perl-parser/src/incremental/reparse.rs b/crates/perl-parser/src/incremental/reparse.rs index 3e1582b23d..dccf390485 100644 --- a/crates/perl-parser/src/incremental/reparse.rs +++ b/crates/perl-parser/src/incremental/reparse.rs @@ -3,6 +3,7 @@ use crate::incremental::{ diagnostics::{LexRestartReport, LexRestartStrategy, ReparseResult}, edit::Edit, lex::{lex_from_live_checkpoint, lex_source_with_checkpoints}, + work::{IncrementalStrategy, IncrementalWorkReceipt}, }; use anyhow::Result; use std::ops::Range; @@ -10,6 +11,7 @@ use std::ops::Range; pub(crate) struct SingleEditReparse { pub(crate) range: Range, pub(crate) lex_restart: LexRestartReport, + pub(crate) fresh_tokens_emitted: usize, pub(crate) token_count: usize, } @@ -62,6 +64,7 @@ pub(crate) fn apply_single_edit( apply_text_edit_to_state(state, edit)?; let lexed = lex_from_live_checkpoint(state.source(), state.line_index(), &live_checkpoint)?; + let fresh_tokens_emitted = lexed.tokens.len(); let mut tokens = state.tokens()[..reused_prefix_tokens].to_vec(); tokens.extend(lexed.tokens); @@ -98,14 +101,17 @@ pub(crate) fn apply_single_edit( Ok(SingleEditReparse { range: restart_byte..state.source().len(), lex_restart, + fresh_tokens_emitted, token_count: state.tokens().len(), }) } pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result { - state.refresh_parse_output(); + let parser_receipt = state.refresh_parse_output(); let source_len = state.source().len(); + let final_node_count = state.parse_output().ast.count_nodes(); let lexed = lex_source_with_checkpoints(state.source(), state.line_index()); + let fresh_tokens_emitted = lexed.tokens.len(); state.replace_lex_state(lexed.tokens, lexed.checkpoints, lexed.stored_checkpoints); let lex_restart = LexRestartReport { @@ -117,132 +123,25 @@ pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result Vec { - let mut lexer = PerlLexer::new(source); - let mut tokens = Vec::new(); - while let Some(token) = lexer.next_token() { - if token.token_type == TokenType::EOF { - break; - } - tokens.push(token); - } - tokens - } - - fn assert_tokens_equal(actual: &[Token], expected: &[Token]) { - assert_eq!(actual.len(), expected.len(), "token count diverged"); - for (index, (actual, expected)) in actual.iter().zip(expected).enumerate() { - assert_eq!(actual.token_type, expected.token_type, "token kind {index}"); - assert_eq!(actual.text, expected.text, "token payload {index}"); - assert_eq!(actual.start, expected.start, "token start {index}"); - assert_eq!(actual.end, expected.end, "token end {index}"); - } - } - - #[test] - fn equal_width_edit_restores_stored_state_without_replaying_old_bytes() -> Result<()> { - let source = "my $x = 1; my $y = 2;"; - let start = source.find("= 1").ok_or_else(|| anyhow::anyhow!("literal missing"))? + 2; - let edit = Edit { - start_byte: start, - old_end_byte: start + 1, - new_end_byte: start + 1, - new_text: "9".to_string(), - }; - let mut state = IncrementalState::new(source.to_string()); - let result = apply_single_edit(&mut state, &edit)?; - - assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); - assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); - assert_eq!(result.lex_restart.reused_suffix_tokens, 0); - assert_eq!(result.range.end, state.source().len()); - assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); - Ok(()) - } - - #[test] - fn method_name_edit_matches_fresh_lex_from_stored_state() -> Result<()> { - let source = "$object->method(); my $x = 1;"; - let start = source.find("method").ok_or_else(|| anyhow::anyhow!("method missing"))?; - let edit = Edit { - start_byte: start, - old_end_byte: start + "method".len(), - new_end_byte: start + "member".len(), - new_text: "member".to_string(), - }; - let mut state = IncrementalState::new(source.to_string()); - let result = apply_single_edit(&mut state, &edit)?; - - assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); - assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); - Ok(()) - } - - #[test] - fn heredoc_body_edit_uses_an_earlier_safe_stored_checkpoint() -> Result<()> { - let source = "my $value = < Result<()> { - let source = "my $a = 1; my $b = 2; my $c = 3;"; - let mut state = IncrementalState::new(source.to_string()); - let first_start = source.find("= 2").ok_or_else(|| anyhow::anyhow!("first edit missing"))? + 2; - let first = Edit { - start_byte: first_start, - old_end_byte: first_start + 1, - new_end_byte: first_start + 1, - new_text: "8".to_string(), - }; - apply_single_edit(&mut state, &first)?; - - let second_start = state - .source() - .find("= 3") - .ok_or_else(|| anyhow::anyhow!("second edit missing"))? + 2; - let second = Edit { - start_byte: second_start, - old_end_byte: second_start + 1, - new_end_byte: second_start + 1, - new_text: "9".to_string(), - }; - let result = apply_single_edit(&mut state, &second)?; - - assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); - assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0); - assert_tokens_equal(state.tokens(), &fresh_tokens(state.source())); - Ok(()) - } } \ No newline at end of file diff --git a/crates/perl-parser/src/incremental/state.rs b/crates/perl-parser/src/incremental/state.rs index a55af8dbc0..c25d49bcbb 100644 --- a/crates/perl-parser/src/incremental/state.rs +++ b/crates/perl-parser/src/incremental/state.rs @@ -1,5 +1,6 @@ use crate::incremental::checkpoint::{LexCheckpoint, ParseCheckpoint, ScopeSnapshot}; use crate::incremental::lex::{StoredLexCheckpoint, lex_source_with_checkpoints}; +use crate::incremental::work::ParserInvocationReceipt; use perl_lexer::Token; use perl_line_index::LineIndex; use perl_parser_core::ast::{Node, NodeKind}; @@ -166,13 +167,23 @@ impl IncrementalState { } /// Refresh the authoritative parser output from the current source. + /// + /// The returned operation-local receipt is created at the same site that + /// invokes `parse_with_recovery`, so a caller cannot claim zero parser work + /// while this production entry point ran. #[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")] - pub(crate) fn refresh_parse_output(&mut self) { + pub(crate) fn refresh_parse_output(&mut self) -> ParserInvocationReceipt { let mut parser = Parser::new(self.source()); let parse_output = parser.parse_with_recovery(); + let nodes_constructed = parse_output.ast.count_nodes(); self.parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast); self.ast = parse_output.ast.clone(); self.parse_output = parse_output; + ParserInvocationReceipt { + full_parser_invocations: 1, + recovery_parser_invocations: 1, + nodes_constructed, + } } pub(crate) fn create_parse_checkpoints(ast: &Node) -> Vec { diff --git a/crates/perl-parser/src/incremental/work.rs b/crates/perl-parser/src/incremental/work.rs new file mode 100644 index 0000000000..b20e42986a --- /dev/null +++ b/crates/perl-parser/src/incremental/work.rs @@ -0,0 +1,285 @@ +use crate::incremental::diagnostics::{LexRestartReport, LexRestartStrategy}; +use thiserror::Error; + +/// Stable production strategy recorded for one incremental result. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum IncrementalStrategy { + /// No source change and no lexer/parser work. + Unchanged, + /// Complete lexer and recovery parser fallback over the final source. + FullFallback, + /// Restore a stored full lexer checkpoint, lex to EOF, then run the full parser. + CheckpointToEofThenFullParse, + /// Restore and synchronize an exact token suffix, then run the full parser. + CheckpointToExactTokenSyncThenFullParse, + /// Patch one exhaustively proven AST leaf without invoking the full parser. + BoundedAstLeafPatch, + /// Non-production comparison after ordinary parsing. + AnalyticalSimilarityOnly, + /// Experimental or unsupported mechanism outside the production authority. + UnsupportedOrExperimental, +} + +impl IncrementalStrategy { + /// Whether this strategy claims to avoid a production full-parser invocation. + #[must_use] + pub const fn claims_no_full_parser(self) -> bool { + matches!(self, Self::Unchanged | Self::BoundedAstLeafPatch) + } +} + +/// Operation-local count returned by the actual canonical parser entry point. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct ParserInvocationReceipt { + pub(crate) full_parser_invocations: usize, + pub(crate) recovery_parser_invocations: usize, + pub(crate) nodes_constructed: usize, +} + +/// Truthful performed-work receipt for one committed incremental result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct IncrementalWorkReceipt { + /// Strategy that produced the committed result. + pub strategy: IncrementalStrategy, + /// Complete production parser invocations. + pub full_parser_invocations: usize, + /// Recovery-aware parser invocations. + pub recovery_parser_invocations: usize, + /// Fresh parser invocations used only as a validation oracle. + pub validation_parser_invocations: usize, + /// Old-source prefix bytes replayed only to reconstruct lexer state. + pub old_prefix_bytes_replayed: usize, + /// Current-source bytes freshly lexed. + pub fresh_bytes_lexed: usize, + /// Lexer tokens freshly emitted by the selected production path. + pub fresh_tokens_emitted: usize, + /// Old prefix tokens retained without fresh lexing. + pub prefix_tokens_retained: usize, + /// Old suffix tokens retained only after exact synchronization. + pub suffix_tokens_retained: usize, + /// AST nodes constructed by the production parser. + pub nodes_constructed: usize, + /// AST nodes retained by reviewed identity, not cloned or compared. + pub nodes_retained_by_identity: usize, + /// AST nodes cloned by the production strategy. + pub nodes_cloned: usize, + /// AST nodes patched in place or in a candidate clone. + pub nodes_patched: usize, + /// AST nodes compared only for analysis or validation. + pub nodes_compared_only: usize, + /// Complete lexer checkpoints restored. + pub checkpoints_restored: usize, + /// Candidate checkpoints rejected or invalidated. + pub checkpoints_invalidated: usize, + /// Complete checkpoints retained in the committed generation. + pub stored_checkpoint_count: usize, + /// Exact final source size. + pub final_source_bytes: usize, + /// Exact final token count. + pub final_token_count: usize, + /// Exact final AST node count. + pub final_node_count: usize, +} + +impl IncrementalWorkReceipt { + pub(crate) fn from_parts( + strategy: IncrementalStrategy, + parser: ParserInvocationReceipt, + lex: LexRestartReport, + fresh_tokens_emitted: usize, + final_source_bytes: usize, + final_token_count: usize, + final_node_count: usize, + ) -> Self { + let checkpoints_restored = usize::from(matches!( + lex.strategy, + LexRestartStrategy::StoredCheckpointToEof + )); + Self { + strategy, + full_parser_invocations: parser.full_parser_invocations, + recovery_parser_invocations: parser.recovery_parser_invocations, + validation_parser_invocations: 0, + old_prefix_bytes_replayed: lex.old_prefix_bytes_replayed, + fresh_bytes_lexed: lex.relexed_bytes, + fresh_tokens_emitted, + prefix_tokens_retained: lex.reused_prefix_tokens, + suffix_tokens_retained: lex.reused_suffix_tokens, + nodes_constructed: parser.nodes_constructed, + nodes_retained_by_identity: 0, + nodes_cloned: 0, + nodes_patched: 0, + nodes_compared_only: 0, + checkpoints_restored, + checkpoints_invalidated: 0, + stored_checkpoint_count: lex.stored_checkpoint_count, + final_source_bytes, + final_token_count, + final_node_count, + } + } + + /// Validate impossible or misleading work combinations. + pub fn validate(&self) -> Result<(), IncrementalWorkReceiptError> { + if self.strategy.claims_no_full_parser() && self.full_parser_invocations != 0 { + return Err(IncrementalWorkReceiptError::HiddenFullParser { + strategy: self.strategy, + invocations: self.full_parser_invocations, + }); + } + if self.prefix_tokens_retained.saturating_add(self.suffix_tokens_retained) + > self.final_token_count + { + return Err(IncrementalWorkReceiptError::RetainedTokenOverflow); + } + if self.nodes_retained_by_identity > self.final_node_count { + return Err(IncrementalWorkReceiptError::RetainedNodeOverflow); + } + if self.suffix_tokens_retained > 0 + && self.strategy != IncrementalStrategy::CheckpointToExactTokenSyncThenFullParse + { + return Err(IncrementalWorkReceiptError::SuffixWithoutExactSync); + } + if self.validation_parser_invocations > 0 + && self.full_parser_invocations == 0 + && self.strategy == IncrementalStrategy::AnalyticalSimilarityOnly + { + return Err(IncrementalWorkReceiptError::AnalysisWithoutProductionResult); + } + match self.strategy { + IncrementalStrategy::Unchanged => { + if self.full_parser_invocations != 0 + || self.recovery_parser_invocations != 0 + || self.fresh_bytes_lexed != 0 + || self.fresh_tokens_emitted != 0 + || self.nodes_constructed != 0 + { + return Err(IncrementalWorkReceiptError::UnchangedPerformedWork); + } + } + IncrementalStrategy::FullFallback => { + if self.full_parser_invocations == 0 + || self.prefix_tokens_retained != 0 + || self.suffix_tokens_retained != 0 + || self.nodes_retained_by_identity != 0 + { + return Err(IncrementalWorkReceiptError::InvalidFullFallback); + } + } + IncrementalStrategy::CheckpointToEofThenFullParse => { + if self.full_parser_invocations == 0 + || self.checkpoints_restored != 1 + || self.old_prefix_bytes_replayed != 0 + || self.suffix_tokens_retained != 0 + { + return Err(IncrementalWorkReceiptError::InvalidCheckpointToEof); + } + } + IncrementalStrategy::CheckpointToExactTokenSyncThenFullParse + | IncrementalStrategy::BoundedAstLeafPatch + | IncrementalStrategy::AnalyticalSimilarityOnly + | IncrementalStrategy::UnsupportedOrExperimental => {} + } + Ok(()) + } +} + +/// Invalid or misleading incremental work receipt. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[non_exhaustive] +pub enum IncrementalWorkReceiptError { + /// A no-full-parser strategy invoked the production parser. + #[error("strategy {strategy:?} invoked the full parser {invocations} time(s)")] + HiddenFullParser { + /// Strategy making the claim. + strategy: IncrementalStrategy, + /// Observed full-parser invocations. + invocations: usize, + }, + /// Retained token counts exceed the final token count. + #[error("retained token counts exceed the final token count")] + RetainedTokenOverflow, + /// Retained node identity count exceeds the final node count. + #[error("retained node count exceeds the final node count")] + RetainedNodeOverflow, + /// Suffix tokens were retained without an exact-sync strategy. + #[error("suffix tokens require exact synchronization")] + SuffixWithoutExactSync, + /// Analysis/oracle work was reported without a production result. + #[error("analysis-only receipt does not identify a production result")] + AnalysisWithoutProductionResult, + /// An unchanged result performed lexer or parser work. + #[error("unchanged strategy reported fresh lexer or parser work")] + UnchangedPerformedWork, + /// Full fallback retained old work or omitted the full parser. + #[error("full fallback receipt is internally inconsistent")] + InvalidFullFallback, + /// Checkpoint-to-EOF receipt omitted restore/full-parse truth or claimed suffix reuse. + #[error("checkpoint-to-EOF receipt is internally inconsistent")] + InvalidCheckpointToEof, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_receipt(strategy: IncrementalStrategy) -> IncrementalWorkReceipt { + IncrementalWorkReceipt { + strategy, + full_parser_invocations: 0, + recovery_parser_invocations: 0, + validation_parser_invocations: 0, + old_prefix_bytes_replayed: 0, + fresh_bytes_lexed: 0, + fresh_tokens_emitted: 0, + prefix_tokens_retained: 0, + suffix_tokens_retained: 0, + nodes_constructed: 0, + nodes_retained_by_identity: 0, + nodes_cloned: 0, + nodes_patched: 0, + nodes_compared_only: 0, + checkpoints_restored: 0, + checkpoints_invalidated: 0, + stored_checkpoint_count: 0, + final_source_bytes: 0, + final_token_count: 0, + final_node_count: 0, + } + } + + #[test] + fn hidden_full_parse_fails_a_no_full_parser_strategy() { + let mut receipt = empty_receipt(IncrementalStrategy::BoundedAstLeafPatch); + receipt.full_parser_invocations = 1; + assert!(matches!( + receipt.validate(), + Err(IncrementalWorkReceiptError::HiddenFullParser { .. }) + )); + } + + #[test] + fn cloning_and_comparison_do_not_become_retained_identity() { + let mut receipt = empty_receipt(IncrementalStrategy::AnalyticalSimilarityOnly); + receipt.full_parser_invocations = 1; + receipt.recovery_parser_invocations = 1; + receipt.nodes_cloned = 8; + receipt.nodes_compared_only = 8; + receipt.final_node_count = 8; + assert_eq!(receipt.nodes_retained_by_identity, 0); + assert!(receipt.validate().is_ok()); + } + + #[test] + fn suffix_retention_requires_the_exact_sync_strategy() { + let mut receipt = empty_receipt(IncrementalStrategy::CheckpointToEofThenFullParse); + receipt.full_parser_invocations = 1; + receipt.recovery_parser_invocations = 1; + receipt.checkpoints_restored = 1; + receipt.suffix_tokens_retained = 1; + receipt.final_token_count = 1; + assert_eq!(receipt.validate(), Err(IncrementalWorkReceiptError::SuffixWithoutExactSync)); + } +} \ No newline at end of file diff --git a/crates/perl-parser/tests/incremental_work_receipts.rs b/crates/perl-parser/tests/incremental_work_receipts.rs new file mode 100644 index 0000000000..ce00109840 --- /dev/null +++ b/crates/perl-parser/tests/incremental_work_receipts.rs @@ -0,0 +1,88 @@ +#![cfg(feature = "incremental")] +//! Public proof for canonical incremental strategy/work receipts. + +use perl_parser::incremental::{IncrementalStrategy, LexRestartStrategy}; +use perl_parser::{Edit, IncrementalState, apply_edits}; + +#[test] +fn unchanged_receipt_reports_zero_fresh_work() -> anyhow::Result<()> { + let source = "my $x = 1;"; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[])?; + + assert_eq!(result.work.strategy, IncrementalStrategy::Unchanged); + assert_eq!(result.work.full_parser_invocations, 0); + assert_eq!(result.work.recovery_parser_invocations, 0); + assert_eq!(result.work.validation_parser_invocations, 0); + assert_eq!(result.work.old_prefix_bytes_replayed, 0); + assert_eq!(result.work.fresh_bytes_lexed, 0); + assert_eq!(result.work.fresh_tokens_emitted, 0); + assert_eq!(result.work.prefix_tokens_retained, state.tokens().len()); + assert_eq!(result.work.suffix_tokens_retained, 0); + assert_eq!(result.work.nodes_constructed, 0); + assert_eq!(result.work.final_source_bytes, source.len()); + assert_eq!(result.work.final_token_count, state.tokens().len()); + assert_eq!(result.work.final_node_count, state.parse_output().ast.count_nodes()); + result.work.validate()?; + Ok(()) +} + +#[test] +fn stored_checkpoint_path_admits_lexer_reuse_but_not_parser_reuse() -> anyhow::Result<()> { + let source = "my $before = 1; my $target = 2; my $after = 3;"; + let start = source.find("= 2").expect("target literal missing") + 2; + let edit = Edit { + start_byte: start, + old_end_byte: start + 1, + new_end_byte: start + 1, + new_text: "9".to_string(), + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof); + assert_eq!(result.work.strategy, IncrementalStrategy::CheckpointToEofThenFullParse); + assert_eq!(result.work.full_parser_invocations, 1); + assert_eq!(result.work.recovery_parser_invocations, 1); + assert_eq!(result.work.validation_parser_invocations, 0); + assert_eq!(result.work.old_prefix_bytes_replayed, 0); + assert_eq!(result.work.fresh_bytes_lexed, result.lex_restart.relexed_bytes); + assert_eq!(result.work.prefix_tokens_retained, result.lex_restart.reused_prefix_tokens); + assert_eq!(result.work.suffix_tokens_retained, 0); + assert_eq!(result.work.nodes_retained_by_identity, 0); + assert_eq!(result.work.nodes_cloned, 0); + assert_eq!(result.work.nodes_patched, 0); + assert_eq!(result.work.nodes_compared_only, 0); + assert_eq!(result.work.checkpoints_restored, 1); + assert_eq!(result.work.final_token_count, state.tokens().len()); + assert_eq!(result.work.final_node_count, state.parse_output().ast.count_nodes()); + assert_eq!(result.work.nodes_constructed, result.work.final_node_count); + result.work.validate()?; + Ok(()) +} + +#[test] +fn oversized_edit_reports_full_fallback_without_retained_work() -> anyhow::Result<()> { + let source = "my $x = 1;"; + let replacement = "my $value = 2;\n".repeat(80); + let edit = Edit { + start_byte: 0, + old_end_byte: source.len(), + new_end_byte: replacement.len(), + new_text: replacement, + }; + let mut state = IncrementalState::new(source.to_string()); + let result = apply_edits(&mut state, &[edit])?; + + assert_eq!(result.work.strategy, IncrementalStrategy::FullFallback); + assert_eq!(result.work.full_parser_invocations, 1); + assert_eq!(result.work.recovery_parser_invocations, 1); + assert_eq!(result.work.prefix_tokens_retained, 0); + assert_eq!(result.work.suffix_tokens_retained, 0); + assert_eq!(result.work.nodes_retained_by_identity, 0); + assert_eq!(result.work.fresh_bytes_lexed, state.source().len()); + assert_eq!(result.work.final_source_bytes, state.source().len()); + assert_eq!(result.work.nodes_constructed, result.work.final_node_count); + result.work.validate()?; + Ok(()) +} \ No newline at end of file diff --git a/scripts/ci/run_parser_integration.py b/scripts/ci/run_parser_integration.py old mode 100644 new mode 100755 index dc20db639f..c297171ec0 --- a/scripts/ci/run_parser_integration.py +++ b/scripts/ci/run_parser_integration.py @@ -116,6 +116,8 @@ def main() -> int: "incremental_parse_output", "--test", "incremental_lexer_restart", + "--test", + "incremental_work_receipts", "--", "--test-threads=4", ] @@ -127,4 +129,4 @@ def main() -> int: if __name__ == "__main__": - raise SystemExit(main()) + raise SystemExit(main()) \ No newline at end of file