Skip to content

instrument(parser): establish strategy/work receipt foundation (#9444) #7332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
EffortlessSteven wants to merge 1 commit into
base: agent/6982-stored-lexer-checkpoints
Choose a base branch
from
Draft

instrument(parser): establish strategy/work receipt foundation (#9444) #7332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions crates/perl-parser/src/incremental/diagnostics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use crate::incremental::work::IncrementalWorkReceipt;
use lsp_types::Diagnostic;
use perl_parser_core::error::ParseOutput;
use std::ops::Range;
Expand Down Expand Up @@ -57,6 +58,8 @@ pub struct ReparseResult {
pub diagnostics: Vec<Diagnostic>,
/// Lexer restart, fresh-work, and token-retention receipt.
pub lex_restart: LexRestartReport,
/// Validated production strategy and performed-work receipt.
pub work: IncrementalWorkReceipt,
/// Number of source bytes covered by parser reparsing work.
pub reparsed_bytes: usize,
/// Compatibility total of old lexer tokens retained from prefix and suffix.
Expand Down
38 changes: 33 additions & 5 deletions crates/perl-parser/src/incremental/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ mod lex;
mod reparse;
mod state;
mod strategy;
mod work;

use anyhow::Result;

Expand All @@ -21,6 +22,10 @@ pub use lex::MAX_STORED_LEX_CHECKPOINTS;
use reparse::{apply_single_edit, apply_text_edit_to_state, full_reparse};
pub use state::IncrementalState;
pub use strategy::MAX_EDIT_SIZE;
pub use work::{
IncrementalStrategy, IncrementalWorkReceipt, IncrementalWorkReceiptError,
};
use work::ParserInvocationReceipt;

pub mod incremental_advanced_reuse;
#[cfg(test)]
Expand Down Expand Up @@ -92,7 +97,7 @@ fn validate_edits(source: &str, edits: &[Edit]) -> Result<usize> {
Ok(total_changed)
}

fn unchanged_result(state: &IncrementalState) -> ReparseResult {
fn unchanged_result(state: &IncrementalState) -> Result<ReparseResult> {
let lex_restart = LexRestartReport {
strategy: LexRestartStrategy::Unchanged,
restart_byte: state.source().len(),
Expand All @@ -102,15 +107,26 @@ fn unchanged_result(state: &IncrementalState) -> ReparseResult {
reused_suffix_tokens: 0,
stored_checkpoint_count: state.stored_lex_checkpoint_count(),
};
ReparseResult {
let work = IncrementalWorkReceipt::from_parts(
IncrementalStrategy::Unchanged,
ParserInvocationReceipt::default(),
lex_restart,
0,
state.source().len(),
state.tokens().len(),
state.parse_output().ast.count_nodes(),

@EffortlessSteven EffortlessSteven Aug 12, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] The Unchanged path walks the entire AST to compute final_node_count, and then clones ParseOutput for the returned result, yet the receipt says zero analysis/validation work and zero cloned nodes. A no-op edit should either reuse stored counts and return an identity-backed snapshot, or account for this traversal and clone. As written, the supposedly zero-work path can be O(AST size) while reporting no work.

);
work.validate()?;
Ok(ReparseResult {
changed_ranges: Vec::new(),
parse_output: state.parse_output().clone(),
diagnostics: Vec::new(),
lex_restart,
work,
reparsed_bytes: 0,
reused_tokens: lex_restart.reused_tokens(),
token_count: state.tokens().len(),
}
})
}

fn apply_text_edits(state: &mut IncrementalState, edits_descending: &[Edit]) -> Result<()> {
Expand All @@ -135,7 +151,7 @@ fn full_reparse_after_edits(
pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result<ReparseResult> {
let total_changed = validate_edits(state.source(), edits)?;
if edits.is_empty() {
return Ok(unchanged_result(state));
return unchanged_result(state);
}

let mut sorted_edits = edits.to_vec();
Expand All @@ -159,13 +175,25 @@ pub fn apply_edits(state: &mut IncrementalState, edits: &[Edit]) -> Result<Repar
Err(_) => return full_reparse_after_edits(state, &sorted_edits),
};

candidate.refresh_parse_output();
let parser_receipt = candidate.refresh_parse_output();
let reused_tokens = reparse.lex_restart.reused_tokens();
let final_node_count = candidate.parse_output().ast.count_nodes();
let work = IncrementalWorkReceipt::from_parts(
IncrementalStrategy::CheckpointToEofThenFullParse,
parser_receipt,
reparse.lex_restart,
reparse.fresh_tokens_emitted,
candidate.source().len(),
reparse.token_count,
final_node_count,
);
work.validate()?;
let result = ReparseResult {
changed_ranges: vec![reparse.range],
parse_output: candidate.parse_output().clone(),
diagnostics: vec![],
lex_restart: reparse.lex_restart,
work,
reparsed_bytes: candidate.source().len(),
reused_tokens,
token_count: reparse.token_count,
Expand Down
137 changes: 18 additions & 119 deletions crates/perl-parser/src/incremental/reparse.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ use crate::incremental::{
diagnostics::{LexRestartReport, LexRestartStrategy, ReparseResult},
edit::Edit,
lex::{lex_from_live_checkpoint, lex_source_with_checkpoints},
work::{IncrementalStrategy, IncrementalWorkReceipt},
};
use anyhow::Result;
use std::ops::Range;

pub(crate) struct SingleEditReparse {
pub(crate) range: Range<usize>,
pub(crate) lex_restart: LexRestartReport,
pub(crate) fresh_tokens_emitted: usize,
pub(crate) token_count: usize,
}

Expand Down Expand Up @@ -62,6 +64,7 @@ pub(crate) fn apply_single_edit(

apply_text_edit_to_state(state, edit)?;
let lexed = lex_from_live_checkpoint(state.source(), state.line_index(), &live_checkpoint)?;
let fresh_tokens_emitted = lexed.tokens.len();

let mut tokens = state.tokens()[..reused_prefix_tokens].to_vec();
tokens.extend(lexed.tokens);
Expand Down Expand Up @@ -98,14 +101,17 @@ pub(crate) fn apply_single_edit(
Ok(SingleEditReparse {
range: restart_byte..state.source().len(),
lex_restart,
fresh_tokens_emitted,
token_count: state.tokens().len(),
})
}

pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result<ReparseResult> {
state.refresh_parse_output();
let parser_receipt = state.refresh_parse_output();
let source_len = state.source().len();
let final_node_count = state.parse_output().ast.count_nodes();
let lexed = lex_source_with_checkpoints(state.source(), state.line_index());
let fresh_tokens_emitted = lexed.tokens.len();
state.replace_lex_state(lexed.tokens, lexed.checkpoints, lexed.stored_checkpoints);

let lex_restart = LexRestartReport {
Expand All @@ -117,132 +123,25 @@ pub(crate) fn full_reparse(state: &mut IncrementalState) -> Result<ReparseResult
reused_suffix_tokens: 0,
stored_checkpoint_count: state.stored_lex_checkpoint_count(),
};
let work = IncrementalWorkReceipt::from_parts(
IncrementalStrategy::FullFallback,
parser_receipt,
lex_restart,
fresh_tokens_emitted,
source_len,
state.tokens().len(),
final_node_count,
);
work.validate()?;

Ok(ReparseResult {
changed_ranges: vec![0..source_len],
parse_output: state.parse_output().clone(),
diagnostics: vec![],
lex_restart,
work,
reparsed_bytes: source_len,
reused_tokens: lex_restart.reused_tokens(),
token_count: state.tokens().len(),
})
}

#[cfg(test)]
mod tests {
use super::*;
use perl_lexer::{PerlLexer, Token, TokenType};

fn fresh_tokens(source: &str) -> Vec<Token> {
let mut lexer = PerlLexer::new(source);
let mut tokens = Vec::new();
while let Some(token) = lexer.next_token() {
if token.token_type == TokenType::EOF {
break;
}
tokens.push(token);
}
tokens
}

fn assert_tokens_equal(actual: &[Token], expected: &[Token]) {
assert_eq!(actual.len(), expected.len(), "token count diverged");
for (index, (actual, expected)) in actual.iter().zip(expected).enumerate() {
assert_eq!(actual.token_type, expected.token_type, "token kind {index}");
assert_eq!(actual.text, expected.text, "token payload {index}");
assert_eq!(actual.start, expected.start, "token start {index}");
assert_eq!(actual.end, expected.end, "token end {index}");
}
}

#[test]
fn equal_width_edit_restores_stored_state_without_replaying_old_bytes() -> Result<()> {
let source = "my $x = 1; my $y = 2;";
let start = source.find("= 1").ok_or_else(|| anyhow::anyhow!("literal missing"))? + 2;
let edit = Edit {
start_byte: start,
old_end_byte: start + 1,
new_end_byte: start + 1,
new_text: "9".to_string(),
};
let mut state = IncrementalState::new(source.to_string());
let result = apply_single_edit(&mut state, &edit)?;

assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof);
assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0);
assert_eq!(result.lex_restart.reused_suffix_tokens, 0);
assert_eq!(result.range.end, state.source().len());
assert_tokens_equal(state.tokens(), &fresh_tokens(state.source()));
Ok(())
}

#[test]
fn method_name_edit_matches_fresh_lex_from_stored_state() -> Result<()> {
let source = "$object->method(); my $x = 1;";
let start = source.find("method").ok_or_else(|| anyhow::anyhow!("method missing"))?;
let edit = Edit {
start_byte: start,
old_end_byte: start + "method".len(),
new_end_byte: start + "member".len(),
new_text: "member".to_string(),
};
let mut state = IncrementalState::new(source.to_string());
let result = apply_single_edit(&mut state, &edit)?;

assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0);
assert_tokens_equal(state.tokens(), &fresh_tokens(state.source()));
Ok(())
}

#[test]
fn heredoc_body_edit_uses_an_earlier_safe_stored_checkpoint() -> Result<()> {
let source = "my $value = <<EOF;\nbody\nEOF\nprint $value;\n";
let start = source.find("body").ok_or_else(|| anyhow::anyhow!("body missing"))?;
let edit = Edit {
start_byte: start,
old_end_byte: start + "body".len(),
new_end_byte: start + "changed".len(),
new_text: "changed".to_string(),
};
let mut state = IncrementalState::new(source.to_string());
let result = apply_single_edit(&mut state, &edit)?;

assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof);
assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0);
assert_eq!(result.lex_restart.reused_suffix_tokens, 0);
assert_tokens_equal(state.tokens(), &fresh_tokens(state.source()));
Ok(())
}

#[test]
fn sequential_edits_regenerate_current_generation_checkpoints() -> Result<()> {
let source = "my $a = 1; my $b = 2; my $c = 3;";
let mut state = IncrementalState::new(source.to_string());
let first_start = source.find("= 2").ok_or_else(|| anyhow::anyhow!("first edit missing"))? + 2;
let first = Edit {
start_byte: first_start,
old_end_byte: first_start + 1,
new_end_byte: first_start + 1,
new_text: "8".to_string(),
};
apply_single_edit(&mut state, &first)?;

let second_start = state
.source()
.find("= 3")
.ok_or_else(|| anyhow::anyhow!("second edit missing"))? + 2;
let second = Edit {
start_byte: second_start,
old_end_byte: second_start + 1,
new_end_byte: second_start + 1,
new_text: "9".to_string(),
};
let result = apply_single_edit(&mut state, &second)?;

assert_eq!(result.lex_restart.strategy, LexRestartStrategy::StoredCheckpointToEof);
assert_eq!(result.lex_restart.old_prefix_bytes_replayed, 0);
assert_tokens_equal(state.tokens(), &fresh_tokens(state.source()));
Ok(())
}
}
13 changes: 12 additions & 1 deletion crates/perl-parser/src/incremental/state.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::incremental::checkpoint::{LexCheckpoint, ParseCheckpoint, ScopeSnapshot};
use crate::incremental::lex::{StoredLexCheckpoint, lex_source_with_checkpoints};
use crate::incremental::work::ParserInvocationReceipt;
use perl_lexer::Token;
use perl_line_index::LineIndex;
use perl_parser_core::ast::{Node, NodeKind};
Expand Down Expand Up @@ -166,13 +167,23 @@ impl IncrementalState {
}

/// Refresh the authoritative parser output from the current source.
///
/// The returned operation-local receipt is created at the same site that
/// invokes `parse_with_recovery`, so a caller cannot claim zero parser work
/// while this production entry point ran.
#[expect(deprecated, reason = "the compatibility AST field mirrors the native parse output")]
pub(crate) fn refresh_parse_output(&mut self) {
pub(crate) fn refresh_parse_output(&mut self) -> ParserInvocationReceipt {
let mut parser = Parser::new(self.source());
let parse_output = parser.parse_with_recovery();
let nodes_constructed = parse_output.ast.count_nodes();

@EffortlessSteven EffortlessSteven Aug 12, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] ast.count_nodes() measures the size of the final retained tree, not how many nodes the parser constructed. Recovery/backtracking may construct and discard additional nodes, so this value cannot support a performed-work claim. The next line also clones the full AST into the compatibility mirror, while the receipt later reports nodes_cloned = 0. Instrument construction and publication cloning at their actual sites, or rename these to final-tree size and make clone work explicit/unknown.

self.parse_checkpoints = Self::create_parse_checkpoints(&parse_output.ast);
self.ast = parse_output.ast.clone();
self.parse_output = parse_output;
ParserInvocationReceipt {
full_parser_invocations: 1,
recovery_parser_invocations: 1,
nodes_constructed,
}
}

pub(crate) fn create_parse_checkpoints(ast: &Node) -> Vec<ParseCheckpoint> {
Expand Down
Loading