Skip to content
Merged
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 90 additions & 16 deletions tools/hygiene/fix-markdown-md032-md026.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,17 @@
# backtick fences cannot interrupt each other — track which fence opened.
_FENCE_OPEN = re.compile(r"^( {0,3})(`{3,}|~{3,})\s*([^`]*)$")

# YAML frontmatter line discriminator: matches a non-empty line that
# starts with a key followed by `:` (with optional leading whitespace
# and optional value). This distinguishes real YAML frontmatter from
# a thematic-break line followed by markdown body. Examples that
# match: `id: B-0001`, `tags:`, `composes_with:`, ` - item`. Note
# ` - item` matches because it could be a YAML list item under a
# key, but in practice a frontmatter never starts on line 1 with a
# list item — the very-first content line should be a key:value.
# The simple `key:` check is enough for the discriminator.
Comment thread
AceHack marked this conversation as resolved.
Outdated
_YAML_KEY_LINE = re.compile(r"^\s*[A-Za-z_][\w-]*\s*:")


def _is_list_or_continuation(line: str) -> bool:
"""Return True if line is a list item or its continuation
Expand All @@ -73,41 +84,104 @@ def _is_list(line: str) -> bool:

def _classify_lines(lines: list[str]) -> list[bool]:
"""Return a boolean list `inside[i]` = True iff line `i` is inside
a fenced code block (and therefore must NOT be touched by the
MD032/MD026 transforms — that would mutate code examples).

A code fence is a line starting with 3+ backticks or 3+ tildes;
closing fence must be the same character class as the opener and
have at least as many characters. We only track the simple case
sufficient for committed-markdown shapes; nested or weird
indentation (>3 spaces makes it a code-indent rather than a fence)
is conservatively treated as "inside" once opened until matching
close — better to skip transforms than to corrupt code."""
inside: list[bool] = []
a region that must NOT be touched by the MD032/MD026 transforms.

Two such regions:

1. **YAML frontmatter** (Jekyll/Hugo/factory-convention shape):
file starts with a line `---`, line 1 is YAML-shaped
(matches `key:` at start), and a closing `---` exists later.
Lines from line 0 through the closing `---` are frontmatter.
Inserting blanks here breaks YAML parsing (e.g.
`composes_with:` followed by blank line then ` - X` parses
as `composes_with: null` plus a separate top-level list).
MD026 would only affect frontmatter if a YAML-key line
happened to match the ATX-heading pattern (`^#+ `) — which
it can't, since YAML keys don't start with `#`. So the YAML
risk is concentrated in MD032's blank-insertion behavior,
and the frontmatter-skip protects that.

2. **Fenced code blocks**: a line starting with 3+ backticks or
3+ tildes; closing fence must be the same character class as
the opener and have at least as many characters. Inserting
blanks here would mutate code examples (e.g. shell-script
with `- option` flags would acquire spurious blanks).

Both regions are conservatively treated as "inside" so transforms
skip them. Better to skip than to corrupt structure.

YAML frontmatter detection is conservative — must distinguish
real frontmatter from a markdown file that happens to start with
a thematic break (`---` followed by content):

Real frontmatter: line 0 is `---`,
line 1 looks YAML-shaped (`key: value` or `key:`),
a closing `---` exists later.
Thematic break: line 0 is `---`,
line 1 is markdown body (heading / prose / etc.).

The YAML-shape check on line 1 is the discriminator. Without it,
a file starting with a horizontal rule would have all subsequent
content marked as "inside frontmatter," skipping every list and
heading from being processed.

Files without frontmatter (line 0 not `---`, or line 1 not
YAML-shaped, or no closing `---`) skip the frontmatter region
entirely — pass-through to the fence-detection logic."""
inside: list[bool] = [False] * len(lines)

# Pass 1: YAML frontmatter region — only if all three conditions:
# (a) line 0 is exactly `---`
# (b) line 1 is YAML-shaped (matches `key:` at start, ignoring
# leading whitespace)
# (c) a closing `---` line exists later
# The (b) check distinguishes real frontmatter from a thematic
# break followed by markdown body.
if (
len(lines) >= 2
and lines[0].rstrip() == "---"
and _YAML_KEY_LINE.match(lines[1])
Comment thread
AceHack marked this conversation as resolved.
):
fm_end = -1
for j in range(2, len(lines)):
if lines[j].rstrip() == "---":
fm_end = j
break
if fm_end > 0:
for k in range(fm_end + 1): # inclusive of closing `---`
inside[k] = True
# If no closing `---` found, conservatively don't mark any
# lines as frontmatter (the file probably isn't real
# frontmatter; treat normally).

# Pass 2: fenced code blocks (skip lines already marked
# inside-frontmatter — they don't open / close fences).
open_char: str | None = None # '`' or '~'
open_len: int = 0
for line in lines:
for i, line in enumerate(lines):
if inside[i]:
continue # Already marked as frontmatter
m = _FENCE_OPEN.match(line)
if m and open_char is None:
# Opening fence
fence = m.group(2)
open_char = fence[0]
open_len = len(fence)
inside.append(True)
inside[i] = True
elif m and open_char is not None:
# Possible closing fence — must be same char class and
# length >= open_len, with no info string.
fence = m.group(2)
if fence[0] == open_char and len(fence) >= open_len and not m.group(3).strip():
inside.append(True) # The closing fence line itself
inside[i] = True # The closing fence line itself
open_char = None
open_len = 0
else:
# A different fence char or shorter — still inside the
# outer block (it's just code that looks fence-shaped).
inside.append(True)
inside[i] = True
else:
inside.append(open_char is not None)
inside[i] = open_char is not None
return inside


Expand Down
Loading