Add additional code comments

core/src/formatter.rs

@@ -62,10 +62,13 @@ impl<'cursor> FileOptions<'cursor> {
     }
 }
 
+/// Cursor position in a Delphi code string, as byte offset from start of string.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub struct Cursor(pub u32);
 
+/// The entrypoint for formatting a string containing Delphi code.
 pub struct Formatter {
+    /// Converts the input string into a token stream.
     lexer: Box<dyn Lexer + Sync>,
     token_consolidators: Vec<Box<dyn RawTokenConsolidator + Sync>>,
     logical_line_parser: Box<dyn LogicalLineParser + Sync>,

core/src/lang.rs

@@ -443,6 +443,7 @@ impl TokenType {
     derive(strum_macros::EnumString),
     strum(ascii_case_insensitive)
 )]
+/// Semantic classification of a logical line after parsing/analysis.
 pub enum LogicalLineType {
     Assignment,
     ConditionalDirective,
@@ -461,20 +462,32 @@ pub enum LogicalLineType {
     CaseArm,
     Declaration,
     VariantRecordCaseArm,
+    /// Comment that should be indented as if part of the parent line (e.g. comments at the end of `if`).
     ParentLineChildComment,
+    /// Catch-all for logical lines whose semantics do not inform special formatting.
     Unknown,
+    /// "Dead" logical lines that are not to be reformatted or reconstructed.
     Voided,
 }
 #[derive(Hash, Debug, Default, Clone, Copy, PartialEq, Eq)]
 pub struct LineParent {
     pub line_index: usize,
+    /// Token (as global token stream index) introducing the child line.
     pub global_token_index: usize,
 }
+
+/// Formatter unit representing one "logical" line in Delphi
+/// (i.e. tokens that could be conceivably formatted on a single line).
+/// Logical lines can be nested.
 #[derive(Debug, PartialEq, Eq)]
 pub struct LogicalLine {
+    /// Parent logical line, if any.
     parent: Option<LineParent>,
+    /// Indentation level relative to parent.
     level: u16,
+    /// Tokens (as global token stream indices) that belong to this logical line.
     tokens: Vec<usize>,
+    /// Semantic meaning of logical line.
     line_type: LogicalLineType,
 }
 impl LogicalLine {
@@ -506,12 +519,15 @@ impl LogicalLine {
     pub fn get_line_type(&self) -> LogicalLineType {
         self.line_type
     }
+    /// Exclude this logical line from reconstruction and remove all its tokens.
+    /// Returns the removed tokens.
     pub fn void_and_drain(&mut self) -> Drain<'_, usize> {
         self.line_type = LogicalLineType::Voided;
         self.tokens.drain(0..)
     }
 }
 
+/// Per-token whitespace metadata used during formatting.
 #[derive(Default, PartialEq, Eq)]
 pub struct FormattingData {
     ignored: bool,
@@ -568,8 +584,9 @@ pub enum MutTokenErr {
     /// The token is ignored, and therefore cannot be mutated.
     TokenIgnored,
 }
-
+/// Token stream with associated mutable formatting metadata for each token.
 pub struct FormattedTokens<'a> {
+    /// Underlying token stream.
     tokens: &'a mut [Token<'a>],
     /// Formatting metadata for each token, with the invariant that the length
     /// will always match the length of [field@FormattedTokens::tokens]
@@ -750,16 +767,24 @@ impl ReconstructionSettings {
 
 pub trait TokenData {
     type TokenType;
+    /// Returns the leading whitespace before the token content.
     fn get_leading_whitespace(&self) -> &str;
+    /// Returns the content of the token, excluding leading whitespace.
     fn get_content(&self) -> &str;
+    /// Returns the type of the token.
     fn get_token_type(&self) -> Self::TokenType;
+    /// Sets the type of the token.
     fn set_token_type(&mut self, typ: Self::TokenType);
 }
 
+/// Lexer token. Consists of the raw slice, info about leading whitespace, and a prospective token type.
 #[derive(Debug, PartialEq, Eq)]
 pub struct RawToken<'a> {
+    /// Corresponding input string slice, including leading whitespace.
     content: &'a str,
+    /// Length of leading whitespace in [`content`].
     ws_len: u32,
+    /// Prospective token type. May be refined by formatting passes.
     token_type: RawTokenType,
 }
 impl<'a> RawToken<'a> {
@@ -790,9 +815,13 @@ impl TokenData for RawToken<'_> {
     }
 }
 
+/// Normalized token used by formatters.
+/// Unlike [`RawToken`], the string content is mutable, so formatting passes can adjust token content.
 #[derive(Debug, PartialEq, Eq)]
 pub struct Token<'a> {
+    /// Text representation of token, including leading whitespace.
     content: Cow<'a, str>,
+    /// Length of leading whitespace in [`content`].
     ws_len: u32,
     token_type: TokenType,
 }

core/src/rules/comment_contents.rs

@@ -2,6 +2,9 @@ use itertools::Itertools;
 
 use crate::prelude::*;
 
+/// Reformats comment contents to a single leading space and no trailing whitespace.
+///
+/// "Separator comments" that repeat a single non-alphanumeric character are excluded.
 pub struct CommentFormatter {}
 
 fn format_line_comment(tok: &mut Token) {

core/src/rules/conditional_directive_consolidator.rs

@@ -9,6 +9,21 @@ use crate::{
 };
 use TokenType as TT;
 
+/// Consolidates simple `{$if} ... {$else} ... {$endif}` blocks into the same logical line,
+/// to support inline expression-style formatting of conditional directives.
+///
+/// ```delphi
+/// // Default behaviour:
+/// var x :=
+/// {$ifdef A}
+///     1
+/// {$else}
+///     2
+/// {$endif};
+///
+/// // With this consolidator:
+/// var x := {$ifdef A} 1 {$else} 2 {$endif};
+/// ```
 pub struct ConditionalDirectiveConsolidator {}
 impl ConditionalDirectiveConsolidator {
     fn is_allowed_token(tokens: &[Token], token_index: usize) -> bool {
@@ -54,6 +69,8 @@ impl ConditionalDirectiveConsolidator {
         let mut new_line_tokens = vec![first_token];
 
         for (&prev, &current) in line.get_tokens().iter().tuple_windows() {
+            // The logical line does not yet contain conditional directives, but we can
+            // detect them indirectly by looking for gaps
             if current - prev > 1 {
                 let gap_start_tok = prev + 1;
                 let gap_end_tok = current - 1;

core/src/rules/deindent_package_directives.rs

@@ -1,5 +1,9 @@
 use crate::prelude::*;
 
+/// Undoes the usual indentation of compiler directives inside package files,
+/// in preference of the Delphi IDE style, which places them all at the margin.
+///
+/// This prevents churn when a pasfmt-formatted file is resaved in the IDE (and vice versa).
 pub struct DeindentPackageDirectives {}
 
 impl LogicalLinesConsolidator for DeindentPackageDirectives {

core/src/rules/eof_newline.rs

@@ -1,5 +1,6 @@
 use crate::{lang::*, traits::LogicalLineFormatter};
 
+/// Ensures that the file ends with a single newline.
 pub struct EofNewline {}
 impl LogicalLineFormatter for EofNewline {
     fn format(&self, formatted_tokens: &mut FormattedTokens<'_>, _input: &LogicalLine) {

core/src/rules/formatting_toggle.rs

@@ -63,6 +63,7 @@ fn parse_toggle(content: &str) -> Option<FormattingToggle> {
     parse_pasfmt_directive_comment_contents(content)
 }
 
+/// Prevents formatting of tokens between `pasfmt off` and `pasfmt on` comments.
 pub struct FormattingToggler {}
 impl TokenIgnorer for FormattingToggler {
     fn ignore_tokens(&self, input: (&[Token], &[LogicalLine]), token_marker: &mut TokenMarker) {

core/src/rules/generics_consolidator.rs

@@ -8,6 +8,9 @@ struct TypeParamState {
     brack_count: u32,
 }
 
+/// Consolidates `<` and `>` tokens that are part of generic type parameter lists into `Generic` chevrons.
+///
+/// Ambiguous cases are assumed to be generic.
 pub struct DistinguishGenericTypeParamsConsolidator;
 impl TokenConsolidator for DistinguishGenericTypeParamsConsolidator {
     fn consolidate(&self, tokens: &mut [Token]) {

core/src/rules/ignore_asm_instructions.rs

@@ -1,5 +1,6 @@
 use crate::prelude::*;
 
+/// Prevents `asm` blocks from being formatted.
 pub struct IgnoreAsmIstructions;
 impl TokenIgnorer for IgnoreAsmIstructions {
     fn ignore_tokens(

core/src/rules/lowercase_keywords.rs

@@ -1,5 +1,6 @@
 use crate::prelude::*;
 
+/// Converts keyword tokens to lowercase.
 pub struct LowercaseKeywords {}
 
 impl LogicalLineFileFormatter for LowercaseKeywords {