diff --git a/core/src/formatter.rs b/core/src/formatter.rs index 37ce7878..b6037742 100644 --- a/core/src/formatter.rs +++ b/core/src/formatter.rs @@ -62,10 +62,13 @@ impl<'cursor> FileOptions<'cursor> { } } +/// Cursor position in a Delphi code string, as byte offset from start of string. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct Cursor(pub u32); +/// The entrypoint for formatting a string containing Delphi code. pub struct Formatter { + /// Converts the input string into a token stream. lexer: Box, token_consolidators: Vec>, logical_line_parser: Box, diff --git a/core/src/lang.rs b/core/src/lang.rs index f74c82a2..bdfdeb5c 100644 --- a/core/src/lang.rs +++ b/core/src/lang.rs @@ -443,6 +443,7 @@ impl TokenType { derive(strum_macros::EnumString), strum(ascii_case_insensitive) )] +/// Semantic classification of a logical line after parsing/analysis. pub enum LogicalLineType { Assignment, ConditionalDirective, @@ -461,20 +462,32 @@ pub enum LogicalLineType { CaseArm, Declaration, VariantRecordCaseArm, + /// Comment that should be indented as if part of the parent line (e.g. comments at the end of `if`). ParentLineChildComment, + /// Catch-all for logical lines whose semantics do not inform special formatting. Unknown, + /// "Dead" logical lines that are not to be reformatted or reconstructed. Voided, } #[derive(Hash, Debug, Default, Clone, Copy, PartialEq, Eq)] pub struct LineParent { pub line_index: usize, + /// Token (as global token stream index) introducing the child line. pub global_token_index: usize, } + +/// Formatter unit representing one "logical" line in Delphi +/// (i.e. tokens that could be conceivably formatted on a single line). +/// Logical lines can be nested. #[derive(Debug, PartialEq, Eq)] pub struct LogicalLine { + /// Parent logical line, if any. parent: Option, + /// Indentation level relative to parent. level: u16, + /// Tokens (as global token stream indices) that belong to this logical line. tokens: Vec, + /// Semantic meaning of logical line. line_type: LogicalLineType, } impl LogicalLine { @@ -506,12 +519,15 @@ impl LogicalLine { pub fn get_line_type(&self) -> LogicalLineType { self.line_type } + /// Exclude this logical line from reconstruction and remove all its tokens. + /// Returns the removed tokens. pub fn void_and_drain(&mut self) -> Drain<'_, usize> { self.line_type = LogicalLineType::Voided; self.tokens.drain(0..) } } +/// Per-token whitespace metadata used during formatting. #[derive(Default, PartialEq, Eq)] pub struct FormattingData { ignored: bool, @@ -568,8 +584,9 @@ pub enum MutTokenErr { /// The token is ignored, and therefore cannot be mutated. TokenIgnored, } - +/// Token stream with associated mutable formatting metadata for each token. pub struct FormattedTokens<'a> { + /// Underlying token stream. tokens: &'a mut [Token<'a>], /// Formatting metadata for each token, with the invariant that the length /// will always match the length of [field@FormattedTokens::tokens] @@ -750,16 +767,24 @@ impl ReconstructionSettings { pub trait TokenData { type TokenType; + /// Returns the leading whitespace before the token content. fn get_leading_whitespace(&self) -> &str; + /// Returns the content of the token, excluding leading whitespace. fn get_content(&self) -> &str; + /// Returns the type of the token. fn get_token_type(&self) -> Self::TokenType; + /// Sets the type of the token. fn set_token_type(&mut self, typ: Self::TokenType); } +/// Lexer token. Consists of the raw slice, info about leading whitespace, and a prospective token type. #[derive(Debug, PartialEq, Eq)] pub struct RawToken<'a> { + /// Corresponding input string slice, including leading whitespace. content: &'a str, + /// Length of leading whitespace in [`content`]. ws_len: u32, + /// Prospective token type. May be refined by formatting passes. token_type: RawTokenType, } impl<'a> RawToken<'a> { @@ -790,9 +815,13 @@ impl TokenData for RawToken<'_> { } } +/// Normalized token used by formatters. +/// Unlike [`RawToken`], the string content is mutable, so formatting passes can adjust token content. #[derive(Debug, PartialEq, Eq)] pub struct Token<'a> { + /// Text representation of token, including leading whitespace. content: Cow<'a, str>, + /// Length of leading whitespace in [`content`]. ws_len: u32, token_type: TokenType, } diff --git a/core/src/rules/comment_contents.rs b/core/src/rules/comment_contents.rs index a1486cc7..849944ab 100644 --- a/core/src/rules/comment_contents.rs +++ b/core/src/rules/comment_contents.rs @@ -2,6 +2,9 @@ use itertools::Itertools; use crate::prelude::*; +/// Reformats comment contents to a single leading space and no trailing whitespace. +/// +/// "Separator comments" that repeat a single non-alphanumeric character are excluded. pub struct CommentFormatter {} fn format_line_comment(tok: &mut Token) { diff --git a/core/src/rules/conditional_directive_consolidator.rs b/core/src/rules/conditional_directive_consolidator.rs index 0309f583..e3e3b96c 100644 --- a/core/src/rules/conditional_directive_consolidator.rs +++ b/core/src/rules/conditional_directive_consolidator.rs @@ -9,6 +9,21 @@ use crate::{ }; use TokenType as TT; +/// Consolidates simple `{$if} ... {$else} ... {$endif}` blocks into the same logical line, +/// to support inline expression-style formatting of conditional directives. +/// +/// ```delphi +/// // Default behaviour: +/// var x := +/// {$ifdef A} +/// 1 +/// {$else} +/// 2 +/// {$endif}; +/// +/// // With this consolidator: +/// var x := {$ifdef A} 1 {$else} 2 {$endif}; +/// ``` pub struct ConditionalDirectiveConsolidator {} impl ConditionalDirectiveConsolidator { fn is_allowed_token(tokens: &[Token], token_index: usize) -> bool { @@ -54,6 +69,8 @@ impl ConditionalDirectiveConsolidator { let mut new_line_tokens = vec![first_token]; for (&prev, ¤t) in line.get_tokens().iter().tuple_windows() { + // The logical line does not yet contain conditional directives, but we can + // detect them indirectly by looking for gaps if current - prev > 1 { let gap_start_tok = prev + 1; let gap_end_tok = current - 1; diff --git a/core/src/rules/deindent_package_directives.rs b/core/src/rules/deindent_package_directives.rs index 6e7f275e..c876374b 100644 --- a/core/src/rules/deindent_package_directives.rs +++ b/core/src/rules/deindent_package_directives.rs @@ -1,5 +1,9 @@ use crate::prelude::*; +/// Undoes the usual indentation of compiler directives inside package files, +/// in preference of the Delphi IDE style, which places them all at the margin. +/// +/// This prevents churn when a pasfmt-formatted file is resaved in the IDE (and vice versa). pub struct DeindentPackageDirectives {} impl LogicalLinesConsolidator for DeindentPackageDirectives { diff --git a/core/src/rules/eof_newline.rs b/core/src/rules/eof_newline.rs index ac5ceb42..3d785f15 100644 --- a/core/src/rules/eof_newline.rs +++ b/core/src/rules/eof_newline.rs @@ -1,5 +1,6 @@ use crate::{lang::*, traits::LogicalLineFormatter}; +/// Ensures that the file ends with a single newline. pub struct EofNewline {} impl LogicalLineFormatter for EofNewline { fn format(&self, formatted_tokens: &mut FormattedTokens<'_>, _input: &LogicalLine) { diff --git a/core/src/rules/formatting_toggle.rs b/core/src/rules/formatting_toggle.rs index 227e353b..2b591957 100644 --- a/core/src/rules/formatting_toggle.rs +++ b/core/src/rules/formatting_toggle.rs @@ -63,6 +63,7 @@ fn parse_toggle(content: &str) -> Option { parse_pasfmt_directive_comment_contents(content) } +/// Prevents formatting of tokens between `pasfmt off` and `pasfmt on` comments. pub struct FormattingToggler {} impl TokenIgnorer for FormattingToggler { fn ignore_tokens(&self, input: (&[Token], &[LogicalLine]), token_marker: &mut TokenMarker) { diff --git a/core/src/rules/generics_consolidator.rs b/core/src/rules/generics_consolidator.rs index 28ca3648..552a0f49 100644 --- a/core/src/rules/generics_consolidator.rs +++ b/core/src/rules/generics_consolidator.rs @@ -8,6 +8,9 @@ struct TypeParamState { brack_count: u32, } +/// Consolidates `<` and `>` tokens that are part of generic type parameter lists into `Generic` chevrons. +/// +/// Ambiguous cases are assumed to be generic. pub struct DistinguishGenericTypeParamsConsolidator; impl TokenConsolidator for DistinguishGenericTypeParamsConsolidator { fn consolidate(&self, tokens: &mut [Token]) { diff --git a/core/src/rules/ignore_asm_instructions.rs b/core/src/rules/ignore_asm_instructions.rs index 41e75baf..460c19e2 100644 --- a/core/src/rules/ignore_asm_instructions.rs +++ b/core/src/rules/ignore_asm_instructions.rs @@ -1,5 +1,6 @@ use crate::prelude::*; +/// Prevents `asm` blocks from being formatted. pub struct IgnoreAsmIstructions; impl TokenIgnorer for IgnoreAsmIstructions { fn ignore_tokens( diff --git a/core/src/rules/lowercase_keywords.rs b/core/src/rules/lowercase_keywords.rs index b1e11d15..3632686f 100644 --- a/core/src/rules/lowercase_keywords.rs +++ b/core/src/rules/lowercase_keywords.rs @@ -1,5 +1,6 @@ use crate::prelude::*; +/// Converts keyword tokens to lowercase. pub struct LowercaseKeywords {} impl LogicalLineFileFormatter for LowercaseKeywords { diff --git a/core/src/rules/optimising_line_formatter/contexts.rs b/core/src/rules/optimising_line_formatter/contexts.rs index 7854bc98..60249ab8 100644 --- a/core/src/rules/optimising_line_formatter/contexts.rs +++ b/core/src/rules/optimising_line_formatter/contexts.rs @@ -74,39 +74,66 @@ pub(super) enum BracketStyle { pub(super) enum ContextType { // Base contexts Base, + /// Inline `var`/`const` declaration (e.g. `begin var A := 1; end`). InlineDeclaration, + /// Raise statement (e.g. `raise `). Raise, + /// "Raise at" statement (e.g. `raise ... at `). RaiseAt, + /// Property declaration header. PropDec, + /// Routine declaration header. RoutineHeader, + /// Line-level container for routine/property directives. DirectivesLine, + /// `for ... in/to/downto ... do` control loop. ForLoop, - // To control the latter of the pair + /// Bracketed region (e.g. `(...)`, `[...]`, `<...>`). Brackets(BracketKind, BracketStyle), // Lists + /// Comma-separated list (e.g. `A, B, C`). CommaList, + /// Element inside a comma-separated list. CommaElem, + /// Semicolon-separated list (e.g. `A; B; C`). SemicolonList, + /// Element inside a semicolon-separated list. SemicolonElem, + /// Sequence of directives (e.g. `abstract; winapi`). DirectiveList, + /// Directive inside a `DirectiveList` (e.g. `abstract`). Directive, // Statement + /// Control-flow statement body/subject container. ControlFlow, + /// Assignment-like statement. Assignment, + /// Typed declaration assignment (e.g. `A: T = ...`). TypedAssignment, // Nested + /// Type expression. Type, + /// Operator binding (higher precedence = lower number). Precedence(u8), + /// `if ... then ... else ...` expression pairing context. IfElse, + /// Left-hand side of assignment-like forms. AssignLHS, + /// Right-hand side of assignment-like forms. AssignRHS, + /// Marks start of control-flow construct. ControlFlowBegin, + /// Conditional directive block (`{$if...}{$else}{$endif}`). ConditionalDirective, - // `MemberAccess` allows non-fluent calls to be on the same line + /// Member-access chain context (e.g. `A.B.C`). MemberAccess, + /// Region related to a keyword (e.g. after `then`, `do`, `of`, `for`). Subject, + /// Control-flow guard expression directly after a leading keyword (e.g. after `if`, `while`). GuardClause, + /// Control-flow guard expression in expression (e.g. in ternaries). ExpressionGuardClause, + /// Anonymous routine header (`function`/`procedure` used inline). AnonHeader, } use ContextType as CT; @@ -129,8 +156,13 @@ pub(super) struct FormattingContext { context_type: ContextType, continuation_delta: u16, starting_token: u32, - /// Used to represent the last token that the formatting requirements will have. - /// Means the indentation can last longer than the rules. + /// Last token index (inclusive) where this context is active for decisions. + /// + /// This is not necessarily the same as when the context disappears from the stack, + /// as sometimes past contexts might need to be inspected to determine future decisions. + /// + /// For example, the RHS of a `=`/`:=` might still read `TypedAssignment`'s state to + /// decide RHS breaking, even though the context itself is no longer active. ending_token: Option, } impl FormattingContext { @@ -275,6 +307,7 @@ pub(super) struct SpecificContextDataStack<'a> { specific_stack: &'a SpecificContextStack<'a>, } impl SpecificContextDataStack<'_> { + /// Return whether the current token in the solution is able to be broken. pub(super) fn parents_support_break(&self) -> bool { self.specific_stack .ctx_data_iter(self.solution) @@ -449,6 +482,7 @@ impl<'a> SpecificContextStack<'a> { } }); + // Locks a break decision in. let apply_pivotal_break = |_: Ref<'_, FormattingContext>, data: &mut FormattingContextState| { data.is_broken |= is_break; @@ -798,6 +832,7 @@ impl<'a> SpecificContextStack<'a> { /// will happen to the contents should it be broken. pub(super) struct LineFormattingContexts<'a> { context_count: usize, + /// Lookup for the stack as of each token index in the line. update_indices: Vec<(u32, NodeRef<'a, FormattingContext>)>, line: &'a LogicalLine, token_types: &'a [TokenType], @@ -807,6 +842,7 @@ impl<'a> LineFormattingContexts<'a> { ParentPointerTree::new(FormattingContext::new(CT::Base)) } + /// Update context_tree with the context stack for a given logical line. pub fn new( line: &'a LogicalLine, token_types: &'a [TokenType], @@ -815,6 +851,7 @@ impl<'a> LineFormattingContexts<'a> { let builder_context_tree = Self::new_tree(); let mut contexts = LineFormattingContextsBuilder::new(&builder_context_tree); + // Init contexts based on what we already know about the logical line match line.get_line_type() { LLT::CaseArm => { contexts.push_utility((CT::ControlFlowBegin, 0)); @@ -853,7 +890,9 @@ impl<'a> LineFormattingContexts<'a> { } } + // History of tokens we've already processed let mut prev_token_types: Vec = Vec::with_capacity(line.get_tokens().len()); + // Get the nth most recent "semantic" token (semantic = not a comment or any directive) macro_rules! last_semantic_token_type { () => { last_semantic_token_type!(0) @@ -866,12 +905,16 @@ impl<'a> LineFormattingContexts<'a> { .nth($i) }; } + + // Remaining token queue let mut next_token_types = line .get_tokens() .iter() .rev() .map(|id| token_types[*id]) .collect::>(); + + // Calculate context stack for all tokens in the line let mut current = next_token_types.pop(); fn next_real_token_type(token_types: &[TokenType]) -> Option { @@ -883,10 +926,12 @@ impl<'a> LineFormattingContexts<'a> { } while let Some(current_token_type) = current { + // For semantic/conditional tokens, first push new contexts that are implied by the previous token. + // This avoids including leading comments/directives in new contexts. + // e.g. for token stream `A , {} B` the `CommaElem` context should start at `B`, not `{}`. if !current_token_type.is_comment_or_compiler_directive() { let last_context_type = contexts.current_context.get().context_type; - // New contexts relating to the previous token are pushed here - // to avoid including any leading comments + // Get most recent semantic OR conditional directive token if let Some(prev_token_type) = prev_token_types .iter() .rev() @@ -1036,9 +1081,11 @@ impl<'a> LineFormattingContexts<'a> { } } } + + // Cache "starting" context for later calculations let last_context_type = contexts.current_context.get().context_type; - // For contexts that apply to the current token + // Push contexts that apply to the current token match current_token_type { TT::Op(OK::LParen | OK::LBrack | OK::LessThan(ChK::Generic)) => { let current_kind = match current_token_type { @@ -1317,10 +1364,13 @@ impl<'a> LineFormattingContexts<'a> { _ => {} } + // Move on to next token trace!("Moving to next token with type: {:?}", current_token_type); contexts.next_token(); + prev_token_types.extend(current); + current = next_token_types.pop(); - // After the current token, some contexts needs to be popped + // Spring cleaning on contexts that are no longer needed match current_token_type { TT::Op(OK::GreaterThan(ChK::Generic)) => { contexts.pop_until_after(context_matches!(CT::Brackets(BracketKind::Angle, _))); @@ -1337,9 +1387,6 @@ impl<'a> LineFormattingContexts<'a> { } _ => {} } - - prev_token_types.extend(current); - current = next_token_types.pop(); } contexts.finalise(); @@ -1352,6 +1399,7 @@ impl<'a> LineFormattingContexts<'a> { } } + /// Convert the context tree into a formatting context lookup by token index (relative to line start). fn write_context_tree( tree: &'a ParentPointerTree, builder: LineFormattingContextsBuilder<'_>, @@ -1410,11 +1458,18 @@ impl<'a> LineFormattingContexts<'a> { /// Facilitates the construction of [`LineFormattingContexts`] incrementally while /// iterating a line's tokens. struct LineFormattingContextsBuilder<'builder> { + /// The tree of formatting contexts for this logical line. contexts: &'builder ParentPointerTree, + /// Lookup for the stack as of each token index in the line. update_indices: Vec<(u32, NodeRef<'builder, FormattingContext>)>, + /// The current top of the context stack. current_context: NodeRef<'builder, FormattingContext>, + /// Utility contexts (provisional contexts) that have not yet been retained. contexts_to_remove: NodeRefSet, + /// Index of the current token in the line. line_index: u32, + /// Precedence(0) contexts that should be rewritten to MemberAccess contexts. + /// This contains all Precedence(0) contexts that have not been marked as [`fluent`]. member_access_contexts: NodeRefSet, } impl<'builder> LineFormattingContextsBuilder<'builder> { @@ -1428,13 +1483,14 @@ impl<'builder> LineFormattingContextsBuilder<'builder> { member_access_contexts: NodeRefSet::new(), } } + /// Iterator over the context types in the stack. fn type_stack(&self) -> impl Iterator + use<'builder> { self.current_context .walk_parents_data() .map(|index| index.context_type) } - /// To ensure the context specified will not be eliminated as unused + /// Ensure the given context will not be eliminated as unused. fn retain(&mut self, context: NodeRef<'builder, FormattingContext>) { trace!( "Retaining context with type: {:?}", @@ -1443,12 +1499,12 @@ impl<'builder> LineFormattingContextsBuilder<'builder> { self.contexts_to_remove.remove(&context); } - /// To ensure the current top of the context stack will not be eliminated as unused + /// Ensure the current top of the context stack will not be eliminated as unused. fn retain_current(&mut self) { self.retain(self.current_context.clone()); } - /// To ensure the first matching context on the stack will not be eliminated as unused + /// Ensure the first matching context on the stack will not be eliminated as unused. fn retain_first(&mut self, filter: F) { let Some(ctx) = self .current_context @@ -1460,7 +1516,7 @@ impl<'builder> LineFormattingContextsBuilder<'builder> { self.retain(ctx); } - /// To indicate the Precedence(0) context is actually a fluent context + /// Mark the given context as a fluent context (i.e. not a member access). fn fluent(&mut self, context: NodeRef<'builder, FormattingContext>) { self.member_access_contexts.remove(&context); } @@ -1488,11 +1544,14 @@ impl<'builder> LineFormattingContextsBuilder<'builder> { node } + /// Push a context onto the top of the stack. fn push>(&mut self, context: C) { self.add_context(context, |context_type| { trace!("Pushing context with type: {context_type:?}") }); } + + /// Push a provisional context that will be removed unless explicitly retained. fn push_utility>(&mut self, context: C) { let context_index = self.add_context(context, |context_type| { trace!("Pushing utility context with type: {context_type:?}") @@ -1501,11 +1560,13 @@ impl<'builder> LineFormattingContextsBuilder<'builder> { } const ADD_ALL_PRECEDENCES: u8 = super::LOWEST_PRECEDENCE + 1; + /// Push utility contexts for all operator precedences, from the given precedence up. fn push_operator_precedences(&mut self, starting_precedence: u8) { for precedence in (super::HIGHEST_PRECEDENCE..starting_precedence).rev() { self.push_utility(CT::Precedence(precedence)); } } + /// Push utility contexts for all operator precedences, from the the current precedence up. fn push_operators(&mut self) { let starting_precedence = match self.type_stack().next() { Some(CT::Precedence(p)) => p, @@ -1513,14 +1574,18 @@ impl<'builder> LineFormattingContextsBuilder<'builder> { }; self.push_operator_precedences(starting_precedence); } + /// Push utility contexts for all operator precedences. fn push_expression(&mut self) { self.push_operator_precedences(Self::ADD_ALL_PRECEDENCES); } + /// Pops the top context off the stack. fn pop(&mut self) { + // Set ending token if unset if self.current_context.get().ending_token.is_none() { self.current_context.get_mut().ending_token = Some(self.line_index.saturating_sub(1)); } + if let Some(node) = self.current_context.parent() { trace!( "Popping context with type: {:?}", @@ -1530,6 +1595,7 @@ impl<'builder> LineFormattingContextsBuilder<'builder> { } } + /// Finds the depth of the first context type that matches the filter, if any. fn find_stack_depth(&mut self, context_filter: F) -> Option { self.type_stack() .enumerate() @@ -1587,6 +1653,7 @@ impl<'builder> LineFormattingContextsBuilder<'builder> { ) { self.retain_current(); } + if self .update_indices .last() @@ -1613,7 +1680,7 @@ impl<'builder> LineFormattingContextsBuilder<'builder> { self.current_context.get_mut() } - /// Finalises the context types and returns which contexts can be removed + /// Perform final processing on the context stack, removing unnecessary contexts and transforming others as needed. fn finalise(&mut self) { // Precedence(0) contexts that have not been deemed as "fluent" are // converted to `MemberAccess` diff --git a/core/src/rules/optimising_line_formatter/mod.rs b/core/src/rules/optimising_line_formatter/mod.rs index 13fa9fa5..3be4d5aa 100644 --- a/core/src/rules/optimising_line_formatter/mod.rs +++ b/core/src/rules/optimising_line_formatter/mod.rs @@ -33,13 +33,14 @@ pub struct OptimisingLineFormatterSettings { pub format_multiline_strings: bool, } +/// Reflows [`LogicalLine`]s within the bounds of the line length limit. pub struct OptimisingLineFormatter { olf_settings: OptimisingLineFormatterSettings, recon_settings: ReconstructionSettings, } /// Realistically, the [`OptimisingLineFormatter`] is a -/// [`LogicalLineFormatter`]. However, it is beneficial to reuse much of the +/// [`LogicalLineFormatter`]. However, it is more performant to reuse much of the /// data constructed in [`OptimisingLineFormatter::format`] across all lines in /// a file. impl LogicalLineFileFormatter for OptimisingLineFormatter { diff --git a/core/src/rules/optimising_line_formatter/requirements.rs b/core/src/rules/optimising_line_formatter/requirements.rs index 80622536..0d8f1b26 100644 --- a/core/src/rules/optimising_line_formatter/requirements.rs +++ b/core/src/rules/optimising_line_formatter/requirements.rs @@ -25,8 +25,8 @@ impl InternalOptimisingLineFormatter<'_, '_> { return DR::Invalid; }; + // Check invariants first for short-circuiting let parents_support_break = contexts_data.parents_support_break(); - if let Some(value) = self.get_formatting_invariant(line_index, line) { return value.map_can_break(parents_support_break); } @@ -317,12 +317,14 @@ impl InternalOptimisingLineFormatter<'_, '_> { ((Some(TT::ConditionalDirective(kind)), Some(TT::Identifier)), DR::Indifferent) if kind.is_end() => { + // Break after `{$endif} ` if the parent context is broken contexts_data .get_last_context(context_matches!(_)) .map(|(_, data)| data.is_child_broken) .if_else_or_default(DR::MustBreak, DR::Indifferent) } ((Some(TT::ConditionalDirective(kind)), Some(_)), DR::Indifferent) if kind.is_end() => { + // Break after `{$endif}` DR::MustBreak } _ => requirement, @@ -330,6 +332,7 @@ impl InternalOptimisingLineFormatter<'_, '_> { requirement.map_can_break(parents_support_break) } + /// Returns the decision that must always be applied for the given token, if any. pub(super) fn get_formatting_invariant( &self, line_index: u32, @@ -382,6 +385,7 @@ impl InternalOptimisingLineFormatter<'_, '_> { } } + /// Get token types of the previous semantic token (`.0`) and current token (`.1`). fn get_token_type_window( &self, line_index: u32, diff --git a/core/src/rules/token_spacing.rs b/core/src/rules/token_spacing.rs index 4a599af8..d227c1d4 100644 --- a/core/src/rules/token_spacing.rs +++ b/core/src/rules/token_spacing.rs @@ -3,6 +3,11 @@ use crate::lang::TokenType as TT; use crate::lang::*; use crate::prelude::*; +/// Normalises spacing between tokens, including: +/// * Single space between operators +/// * One space before comments +/// * One space before and after keywords +/// * etc. pub struct TokenSpacing {} impl LogicalLineFileFormatter for TokenSpacing { fn format(&self, formatted_tokens: &mut FormattedTokens, _input: &[LogicalLine]) { diff --git a/core/src/traits.rs b/core/src/traits.rs index d4073d96..4c93b5d8 100644 --- a/core/src/traits.rs +++ b/core/src/traits.rs @@ -1,36 +1,45 @@ use crate::formatter::{Cursor, TokenMarker}; use crate::lang::*; +/// Formatting stage that reads a code string into a vector of raw tokens. pub trait Lexer { fn lex<'a>(&self, input: &'a str) -> Vec>; } +/// Formatting stage that adjusts raw tokens. pub trait RawTokenConsolidator { fn consolidate(&self, tokens: &mut [RawToken]); } + +/// Formatting stage that adjusts tokens. pub trait TokenConsolidator { fn consolidate(&self, tokens: &mut [Token]); } +/// Formatting stage that parses a stream of tokens into logical lines. pub trait LogicalLineParser { fn parse<'a>(&self, input: Vec>) -> (Vec, Vec>); } +/// Formatting stage that converts or consolidates logical lines into a different representation. pub trait LogicalLinesConsolidator { fn consolidate(&self, input: (&mut [Token], &mut [LogicalLine])); } +/// Formatting stage that marks certain tokens to be ignored during formatting. pub trait TokenIgnorer { fn ignore_tokens(&self, input: (&[Token], &[LogicalLine]), token_marker: &mut TokenMarker); } - +/// Formatting stage that marks certain tokens to be removed during formatting. pub trait TokenRemover { fn remove_tokens(&self, input: (&[Token], &[LogicalLine]), token_marker: &mut TokenMarker); } +/// Formatting stage that adjusts token formatting metadata for a single logical line. pub trait LogicalLineFormatter { fn format(&self, formatted_tokens: &mut FormattedTokens<'_>, input: &LogicalLine); } +/// Formatting stage that adjusts token formatting metadata for all logical lines. pub trait LogicalLineFileFormatter { fn format(&self, formatted_tokens: &mut FormattedTokens<'_>, input: &[LogicalLine]); } @@ -40,6 +49,7 @@ pub trait CursorTracker { fn notify_token_deleted(&mut self, deleted_token: usize); } +/// Formatting stage that reconstructs a formatted token stream back into a Delphi code string. pub trait LogicalLinesReconstructor { fn reconstruct(&self, formatted_tokens: FormattedTokens, out: &mut String);