feat(action-grammar): treat ?/*/+ as special quantifier chars

ts/extensions/agr-language/sample.agr

@@ -48,7 +48,7 @@ import { Ordinal, CalendarDate };
             artists: [artist]
         }
     }
-  | pause (the)? music? -> { actionName: "pause" };
+  | pause (the)? (music)? -> { actionName: "pause" };
 
 // ─── Exported rule ────────────────────────────────────────────────────────────
 
@@ -160,15 +160,21 @@ export <PublicRule> = hello | goodbye;
 // ─── Operators and grouping ──────────────────────────────────────────────────
 
 <Operators> =
-    one? two* three+ four  // optional, zero-or-more, one-or-more
+    (one)? (two)* (three)+ four  // optional, zero-or-more, one-or-more
   | (first | second | third)  // alternation with grouping
   | (item)+ // one-or-more group
   | (prefix)* suffix; // zero-or-more group
+  // Literal "?" must be escaped (bare ?/*/+ are quantifiers after ) or > only):
+  //   what is the time\?                 // literal trailing ?
+  //   who sings song <SongName>\?        // required Song + literal ?
+  //   who sings song <SongName>?         // OPTIONAL Song, NO literal "?" (silent pitfall)
+  //   who sings song (<SongName>)?\?     // optional Song + literal ?
+  // Writer prefers (<Name>)? over bare <Name>?.
 
 // ─── Optional capture quantifiers ────────────────────────────────────────────
 
 <ItemList> =
-    add $(item:word)+ to $(list:string) -> {
+    add ($(item:word))+ to $(list:string) -> {
         actionName: "addItems",
         parameters: { items: [item], listName: list }
     };

ts/packages/actionGrammar/README.md

@@ -42,6 +42,13 @@ import { Helper } from "./other.agr";                 // Grammar imports
 <SkipTrack> = (skip | next) (track | song)?            // Optionals, alternation
     -> { actionName: "skip" };
 <Items> = $(item:string) (, $(item:string))*;          // Repetition (Kleene star)
+// Quantifiers ? * + are special: valid only after ")" or ">" (e.g. (<Polite>)?, <Song>?).
+// Bare ? elsewhere is a parse error — escape literals as \?
+// what is the time\?                                 // literal trailing ?
+// who sings song <Song>\?                            // required Song + literal ?
+// who sings song <Song>?                             // OPTIONAL Song, NO literal "?" (pitfall)
+// who sings song (<Song>)?\?                         // optional Song + literal "?"
+// Writer/prettier always emits the grouped form: (<Name>)? not bare <Name>?.
 ```
 
 ## Exports

ts/packages/actionGrammar/src/builtInGrammarCategories.ts

@@ -10,7 +10,8 @@
  * the stored grammar is fully self-contained.
  *
  * Naming convention for prompt use: <CategoryName>
- * Usage in patterns: (<CategoryName>)?   (note: (<Name>)? not <Name>? — bare optional not yet supported)
+ * Usage in patterns: (<CategoryName>)? or bare <CategoryName>?
+ * (writer/prettier always emits the grouped form)
  */
 export interface BuiltInGrammarCategory {
     /** AGR rule name — used as <Name> in patterns */

ts/packages/actionGrammar/src/dfaMatcher.ts

@@ -14,6 +14,7 @@ import { globalPhraseSetRegistry } from "./builtInPhraseMatchers.js";
 import {
     normalizeToken,
     parseNumberToken,
+    tokenizeRequestKeepingTrailingPunct,
     tokenizeRequestWithOffsets,
 } from "./nfaMatcher.js";
 import type { GrammarCompletionResult } from "./grammarCompletion.js";
@@ -720,12 +721,17 @@ function compareDFAMatchPriority(a: DFAMatchResult, b: DFAMatchResult): number {
 }
 
 /**
- * Match tokens against a DFA, performing a two-pass split-candidate strategy
- * when the DFA has split candidates (for spacing=optional/auto grammars).
+ * Match tokens against a DFA, performing a multi-pass strategy:
  *
- * Pass 1 — original whitespace tokens.
+ * Pass 1 — original whitespace tokens (trailing sentence punct stripped).
  * Pass 2 — pre-split tokens using dfa.splitCandidates (e.g. "Swift's" → ["Swift", "'s"]).
- * The higher-priority result is returned.
+ * Pass 3 — when `spacingContext.request` is set and earlier passes miss, retry
+ *           with trailing sentence punctuation peeled into its own tokens so
+ *           grammars with a standalone literal `\?` match natural questions
+ *           like `"who sings song hello?"` (mirrors matchGrammarWithNFA).
+ *
+ * The higher-priority result across passes is returned.  The strip pass is
+ * preferred when it matches (trailing punct as flex-space).
  */
 export function matchDFAWithSplitting(
     dfa: DFA,
@@ -739,8 +745,47 @@ export function matchDFAWithSplitting(
      * original request the matcher cannot distinguish "  helloworld" from
      * "helloworld".  Callers that have the request and grammar should
      * pass them; legacy callers can omit.
+     *
+     * When `request` is provided, a failed strip-tokenized match is retried
+     * with trailing punctuation kept as separate tokens (see Pass 3 above).
      */
-    spacingContext?: { request: string; grammar: Grammar },
+    spacingContext?: { request: string; grammar?: Grammar },
+): DFAMatchResult {
+    const best = matchDFAWithSplittingCore(dfa, tokens, debugMode);
+
+    // Pass 3: peel glued trailing sentence punctuation (needs original request).
+    if (!best.matched && spacingContext?.request) {
+        const punctTokens = tokenizeRequestKeepingTrailingPunct(
+            spacingContext.request,
+        );
+        if (
+            punctTokens.length > 0 &&
+            (punctTokens.length !== tokens.length ||
+                punctTokens.some((t, i) => t !== tokens[i]))
+        ) {
+            const punctBest = matchDFAWithSplittingCore(
+                dfa,
+                punctTokens,
+                debugMode,
+            );
+            if (punctBest.matched) {
+                return applySpacingNoneRejection(
+                    punctBest,
+                    punctTokens,
+                    spacingContext,
+                );
+            }
+        }
+    }
+
+    return applySpacingNoneRejection(best, tokens, spacingContext);
+}
+
+/** Split-candidate passes only (no trailing-punct retry, no spacing=none). */
+function matchDFAWithSplittingCore(
+    dfa: DFA,
+    tokens: string[],
+    debugMode: boolean,
 ): DFAMatchResult {
     // O(1) first-token pre-filter
     if (dfaFirstTokenRejects(dfa, tokens)) {
@@ -776,10 +821,21 @@ export function matchDFAWithSplitting(
             }
         }
     }
+    return best;
+}
 
+function applySpacingNoneRejection(
+    best: DFAMatchResult,
+    tokens: string[],
+    spacingContext?: { request: string; grammar?: Grammar },
+): DFAMatchResult {
     // spacing=none rejection of leading/trailing whitespace.  Mirrors the
     // check in matchGrammarWithNFA (nfaMatcher.ts).
-    if (best.matched && spacingContext && best.ruleIndex !== undefined) {
+    if (
+        best.matched &&
+        spacingContext?.grammar &&
+        best.ruleIndex !== undefined
+    ) {
         const { request, grammar } = spacingContext;
         const hasOuterWhitespace =
             request.length !== request.trim().length &&

ts/packages/actionGrammar/src/fuzz/grammarGenerator.ts

@@ -861,25 +861,25 @@ function maybeEscapeWord(
  *     space character as part of the token, not a separator.
  *
  * Grammar-special chars (`|`, `(`, `)`, `<`, `>`, `$`, `-`, `;`,
- * `{`, `}`, `[`, `]`, `\`) and comment starters (`/`) are
- * excluded - they require backslash escapes that the parser handles
- * via the broader `escapeProb` knob, not via embedded-separator
- * semantics.
+ * `{`, `}`, `[`, `]`, `?`, `*`, `+`, `\`) and comment starters (`/`) are
+ * excluded from bare embedding — `?`/`*`/`+` require escapes in source
+ * (they are postfix quantifiers). Entries below that are special use the
+ * escaped source form.
  */
 const SEPARATOR_LITERAL_CHARS: ReadonlyArray<readonly [string, string]> = [
     [",", ","],
     [".", "."],
     [":", ":"],
     ["!", "!"],
-    ["?", "?"],
+    ["\\?", "?"],
     ["=", "="],
     ["@", "@"],
     ["#", "#"],
     ["%", "%"],
     ["&", "&"],
     ["'", "'"],
     ['"', '"'],
-    ["+", "+"],
+    ["\\+", "+"],
     // Escaped space: source `\ ` decodes to a single space char that
     // is treated as part of the literal (not a flex-space).
     ["\\ ", " "],

ts/packages/actionGrammar/src/generation/scenarioBasedGenerator.ts

@@ -589,10 +589,9 @@ export class ScenarioBasedGrammarGenerator {
                 output += `<${categoryName}> =\n`;
                 verbsList.forEach((verb, index) => {
                     const separator = index < verbsList.length - 1 ? " |" : "";
-                    // Escape backslashes first, then single quotes in verb phrases
-                    const escapedVerb = verb
-                        .replace(/\\/g, "\\\\")
-                        .replace(/'/g, "\\'");
+                    // Same escapes as generatePrefixSuffixRule — quotes are
+                    // ordinary match chars, so bare ?/*/+ inside would parse-error.
+                    const escapedVerb = this.escapeSpecialChars(verb);
                     output += `    '${escapedVerb}'${separator}\n`;
                 });
                 output += `\n`;
@@ -856,13 +855,13 @@ export class ScenarioBasedGrammarGenerator {
 
     /**
      * Escape special characters in quoted string literals
-     * Special chars: \, @, |, (, ), <, >, $, -, {, }, [, ], '
+     * Special chars: \, @, |, (, ), <, >, $, -, {, }, [, ], ?, *, +, '
      * Backslashes must be escaped first to avoid double-escaping
      */
     private escapeSpecialChars(text: string): string {
         return text
             .replace(/\\/g, "\\\\")
-            .replace(/[@|()\[\]<>$\-{}']/g, "\\$&");
+            .replace(/[@|()\[\]<>$\-{}?*+']/g, "\\$&");
     }
 
     /**

ts/packages/actionGrammar/src/generation/schemaToGrammarGenerator.ts

@@ -47,9 +47,9 @@ The Action Grammar format uses:
 - Rule definitions: <RuleName> = pattern;
 - Literal text: "play" or 'play'
 - Wildcards with types: $(name:Type) - captures any text and assigns it to 'name' with validation type 'Type'
-- Optional elements: element?
-- Zero or more: element*
-- One or more: element+
+- Optional: (element)? or <Name>?   (quantifiers ? * + ONLY valid immediately after ")" or ">")
+- Zero or more: (element)* or <Name>*
+- One or more: (element)+ or <Name>+
 - Alternation: pattern1 | pattern2
 - Grouping: (expression) - groups expressions for operators
 - Rule references: <RuleName>
@@ -62,8 +62,9 @@ FULL EXAMPLE showing captures and action body:
 
 CRITICAL SYNTAX RULES:
 1. ALWAYS use parentheses around alternatives when combined with operators
-   CORRECT: ('can you'? 'add' | 'include')
-   WRONG: 'can you'? 'add' | 'include'
+   CORRECT: (((can you)? add) | include)
+   WRONG: can you? add | include   (bare ? after a word is a PARSE ERROR)
+   WRONG: 'can you'? 'add' | 'include'   (bare ? after a string is also a PARSE ERROR)
 
 2. ALWAYS use parentheses around groups that should be treated as a unit
    CORRECT: ('on' | 'for') $(date:CalendarDate)
@@ -115,7 +116,15 @@ CRITICAL SYNTAX RULES:
    CORRECT: // This is a comment
    WRONG:   # This is a comment
 
-10. Hyphenated and apostrophe string literals:
+10. Quantifiers ? * + are SPECIAL characters in patterns:
+    - Valid ONLY immediately after ")" or ">" : (<Polite>)?, <Song>?, $(x)?, (a|b)*
+    - Bare "?" after a word is a PARSE ERROR. Escape literals: what is the time\\?
+    - Required name + question mark: who sings song <Song>\\?
+    - Optional name + question mark: who sings song (<Song>)?\\?
+    - Do NOT write <Polite>? intending a literal "?"; that makes Polite optional.
+    - The writer/prettier prefers the grouped form (<Name>)? over bare <Name>?.
+
+11. Hyphenated and apostrophe string literals:
     a) Apostrophes/contractions: "don't" "it's" "let's"  (NOT 'don\'t' or 'it\'s' — use double quotes)
     b) Hyphenated words like 'auto-reload' or "auto-generate" CANNOT appear in any quoted string
        (hyphens are special characters even inside double-quoted strings).
@@ -124,13 +133,13 @@ CRITICAL SYNTAX RULES:
        CORRECT: ('auto' 'generate' | 'autogenerate')?
        WRONG:   'auto-generate' or "auto-generate"  (both cause parse errors!)
 
-11. Action body values must be SIMPLE variable names only — no dot notation, array access, or expressions:
+12. Action body values must be SIMPLE variable names only — no dot notation, array access, or expressions:
     CORRECT: -> { actionName: "create", parameters: { name: name, language: language } }
     WRONG:   -> { actionName: "create", parameters: { declaration: details.declaration, body: details.body } }
     If a TypeScript schema parameter has nested fields, just capture it as a single string wildcard.
     Grammar rules capture flat key/value pairs; don't model nested object structures.
 
-12. When using a CUSTOM SUB-RULE (not a built-in entity type) as a wildcard type, wrap the rule name in angle brackets:
+13. When using a CUSTOM SUB-RULE (not a built-in entity type) as a wildcard type, wrap the rule name in angle brackets:
     CORRECT: $(location:<LocationSpec>)   — rule reference in wildcard (angle brackets required)
     CORRECT: $(days:<DaysSpec>)?          — optional rule-typed capture
     WRONG:   $(location:LocationSpec)     — rule name without angle brackets (will cause "Undefined type" error)
@@ -145,7 +154,8 @@ EFFICIENCY GUIDELINES:
    Example: If multiple actions use date expressions, create <DateExpr> = ('on' | 'for') $(date:CalendarDate);
 
 2. Create shared vocabulary rules for common phrases
-   Example: <Polite> = 'can you'? | 'please'? | 'would you'?;
+   Example: <Polite> = can you | please | would you;
+   Then make the whole rule optional at use sites: (<Polite>)? open outlook
 
 3. Reuse entity type rules across actions
    Example: If multiple actions need participant names, reference the same wildcard pattern
@@ -201,14 +211,27 @@ IMPROVEMENT INSTRUCTIONS:
 AVAILABLE ENTITY TYPES AND CONVERTERS:
 {entityTypes}
 
+CRITICAL SYNTAX RULES (must follow when extending):
+1. Quantifiers ? * + are SPECIAL characters in patterns:
+   - Valid ONLY immediately after ")" or ">" : (<Polite>)?, <Song>?, $(x)?, (a|b)*
+   - Bare "?" after a word is a PARSE ERROR. Escape literals: what is the time\\?
+   - Required name + question mark: who sings song <Song>\\?
+   - Optional name + question mark: who sings song (<Song>)?\\?
+   - Do NOT write <Polite>? intending a literal "?"; that makes Polite optional.
+   - The writer/prettier prefers the grouped form (<Name>)? over bare <Name>?.
+   CORRECT: (please)?  (can you)?  (<Polite>)?  <Song>?  $(x)?
+   WRONG:   please?  can you?  'can you'?  "please"?  word*
+2. Comments use // not #
+3. Action rule names MUST match the exact action name (not capitalized)
+
 Your task:
 1. Analyze the existing grammar and identify areas for improvement
 2. Incorporate the new examples by extending or refining existing rules
 3. Follow the improvement instructions to enhance the grammar
 4. Maintain consistency with existing patterns and style
 5. Ensure all actions in the schema are covered
 6. Keep shared sub-rules and don't duplicate patterns
-7. Follow all AGR syntax rules (see above)
+7. Follow all AGR syntax rules above (especially quantifier special-char rules)
 8. IMPORTANT: Use exact action names for action rules (e.g., <scheduleEvent> = ... ;, not <ScheduleEvent> = ... ;)
    This enables easy targeting of specific actions when extending grammars incrementally
 
@@ -639,6 +662,11 @@ Remember the CRITICAL SYNTAX RULES:
     CORRECT: { name: name, language: language }
     WRONG:   { declaration: details.declaration }  (dot notation not valid)
     Capture each parameter as its own $(var:type) wildcard.
+12. Quantifiers ? * + are SPECIAL — valid ONLY immediately after ")" or ">":
+    CORRECT: (<Polite>)?, <Song>?, $(x)?, (a|b)*, (please)?, (can you)?
+    WRONG:   please?  can you?  'can you'?  "please"?  word*  (bare quantifier = PARSE ERROR)
+    Literal ? * + require backslash escapes: what is the time\\?
+    Prefer grouped form (<Name>)? over bare <Name>?.
 
 Return the complete corrected grammar, starting with the copyright header.`;