Memory Optimizations

CHANGES.md

@@ -120,6 +120,7 @@
   splicing the unchanged blocks into each parent's child list in a single pass rather
   than removing and re-inserting each one, which rescanned and shifted the whole child
   list on every conversion (#5213)
+- Reduce memory requirement for Node and Leaf class (#5222)
 
 ### Output
 

src/black/brackets.py

@@ -56,7 +56,7 @@ class BracketMatchError(Exception):
     """Raised when an opening bracket is unable to be matched to a closing bracket."""
 
 
-@dataclass
+@dataclass(slots=True, eq=False)
 class BracketTracker:
     """Keeps track of brackets on a line."""
 
@@ -156,7 +156,7 @@ def delimiter_count_with_priority(self, priority: Priority = 0) -> int:
             return 0
 
         priority = priority or self.max_delimiter_priority()
-        return sum(1 for p in self.delimiters.values() if p == priority)
+        return sum(p == priority for p in self.delimiters.values())
 
     def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
         """In a for loop, or comprehension, the variables are often unpacks.

src/black/comments.py

@@ -2,7 +2,7 @@
 from collections.abc import Collection, Iterator
 from dataclasses import dataclass
 from functools import lru_cache
-from typing import Final, Union
+from typing import Final
 
 from black.mode import Mode
 from black.nodes import (
@@ -22,7 +22,7 @@
 from blib2to3.pytree import Leaf, Node
 
 # types
-LN = Union[Leaf, Node]
+LN = Leaf | Node
 
 FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"}
 FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"}
@@ -37,7 +37,7 @@
 _COMMENT_LIST_SEPARATOR = ";"
 
 
-@dataclass
+@dataclass(slots=True, eq=False, repr=False)
 class ProtoComment:
     """Describes a piece of syntax that is a comment.
 

src/black/linegen.py

@@ -1054,13 +1054,12 @@ def _first_right_hand_split(
                     len(str(leaf))
                     for leaf in hugged_opening_leaves + hugged_closing_leaves
                 )
-                if is_line_short_enough(
+
+                # Do not hug if it fits on a single line.
+                should_hug = not is_line_short_enough(
                     inner_body, mode=replace(line.mode, line_length=line_length)
-                ):
-                    # Do not hug if it fits on a single line.
-                    should_hug = False
-                else:
-                    should_hug = True
+                )
+
             if should_hug:
                 body_leaves = inner_body_leaves
                 head_leaves.extend(hugged_opening_leaves)

src/black/lines.py

@@ -38,7 +38,7 @@
 LN = Union[Leaf, Node]
 
 
-@dataclass
+@dataclass(slots=True, eq=False)
 class Line:
     """Holds leaves and comments. Can be printed with `str(line)`."""
 
@@ -197,11 +197,7 @@ def _is_triple_quoted_string(self) -> bool:
         if not self or self.leaves[0].type != token.STRING:
             return False
         value = self.leaves[0].value
-        if value.startswith(('"""', "'''")):
-            return True
-        if value.startswith(("r'''", 'r"""', "R'''", 'R"""')):
-            return True
-        return False
+        return value.startswith(('"""', "'''", "r'''", 'r"""', "R'''", 'R"""'))
 
     @property
     def is_docstring(self) -> bool:
@@ -483,22 +479,21 @@ def __str__(self) -> str:
             return "\n"
 
         indent = "    " * self.depth
-        leaves = iter(self.leaves)
-        first = next(leaves)
-        res = f"{first.prefix}{indent}{first.value}"
-        res += "".join(str(leaf) for leaf in leaves)
-        comments_iter = itertools.chain.from_iterable(self.comments.values())
-        comments = [str(comment) for comment in comments_iter]
-        res += "".join(comments)
+        first = self.leaves[0]
+
+        rest_str = "".join(map(str, self.leaves[1:]))
+        comments_str = "".join(
+            str(comment) for group in self.comments.values() for comment in group
+        )
 
-        return res + "\n"
+        return f"{first.prefix}{indent}{first.value}{rest_str}{comments_str}\n"
 
     def __bool__(self) -> bool:
         """Return True if the line has leaves or comments."""
         return bool(self.leaves or self.comments)
 
 
-@dataclass
+@dataclass(slots=True, eq=False, repr=False)
 class RHSResult:
     """Intermediate split result from a right hand split."""
 
@@ -509,7 +504,7 @@ class RHSResult:
     closing_bracket: Leaf
 
 
-@dataclass
+@dataclass(slots=True, eq=False, repr=False)
 class LinesBlock:
     """Class that holds information about a block of formatted lines.
 
@@ -549,7 +544,7 @@ class _DecoratedFuncInfo(NamedTuple):
     is_multi: bool
 
 
-@dataclass
+@dataclass(slots=True, eq=False, repr=False)
 class EmptyLineTracker:
     """Provides a stateful method that returns the number of potential extra
     empty lines needed before and after the currently processed line.
@@ -1610,7 +1605,6 @@ def can_omit_invisible_parens(
         # a leading opening bracket and a trailing closing bracket.  If the
         # opening bracket doesn't match our rule, maybe the closing will.
 
-    penultimate = line.leaves[-2]
     last = line.leaves[-1]
 
     if (
@@ -1624,6 +1618,7 @@ def can_omit_invisible_parens(
             and last.parent.type != syms.trailer
         )
     ):
+        penultimate = line.leaves[-2]
         if penultimate.type in OPENING_BRACKETS:
             # Empty brackets don't help.
             return False

src/black/output.py

@@ -89,8 +89,7 @@ def diff(a: str, b: str, a_name: str, b_name: str) -> str:
         if line[-1] == "\n":
             diff_lines.append(line)
         else:
-            diff_lines.append(line + "\n")
-            diff_lines.append("\\ No newline at end of file\n")
+            diff_lines.extend((line + "\n", "\\ No newline at end of file\n"))
     return "".join(diff_lines)
 
 

src/black/ranges.py

@@ -525,7 +525,7 @@ def _get_line_range(node_or_nodes: LN | list[LN]) -> set[int]:
                 return set()
 
 
-@dataclass
+@dataclass(slots=True, eq=False, repr=False)
 class _LinesMapping:
     """1-based lines mapping from original source to modified source.
 

src/black/trans.py

@@ -360,7 +360,7 @@ def _get_key(string: str) -> CustomSplitMapKey:
             A unique identifier that is used internally to map @string to a
             group of custom splits.
         """
-        return (id(string), string)
+        return id(string), string
 
     def add_custom_splits(
         self, string: str, custom_splits: Iterable[CustomSplit]
@@ -422,6 +422,8 @@ class StringMerger(StringTransformer, CustomSplitMapMixin):
         StringMerger provides custom split information to StringSplitter.
     """
 
+    __slots__ = ()
+
     def do_match(self, line: Line) -> TMatchResult:
         LL = line.leaves
 
@@ -1016,13 +1018,11 @@ def do_transform(
             string_parser = StringParser()
             rpar_idx = string_parser.parse(LL, string_idx)
 
-            should_transform = True
-            for leaf in (LL[string_idx - 1], LL[rpar_idx]):
-                if line.comments_after(leaf):
-                    # Should not strip parentheses which have comments attached
-                    # to them.
-                    should_transform = False
-                    break
+            # Should not strip parentheses which have comments attached
+            # to them.
+            should_transform = not any(
+                line.comments_after(leaf) for leaf in (LL[string_idx - 1], LL[rpar_idx])
+            )
             if should_transform:
                 string_and_rpar_indices.extend((string_idx, rpar_idx))
 
@@ -1358,7 +1358,7 @@ def iter_fexpr_spans(s: str) -> Iterator[tuple[int, int]]:
             j = stack.pop()
             # we've made it back out of the expression! yield the span
             if not stack:
-                yield (j, i + 1)
+                yield j, i + 1
             i += 1
             continue
 
@@ -1397,11 +1397,13 @@ def _toggle_fexpr_quotes(fstring: str, old_quote: str) -> str:
     escaping, once Black figures out how to parse the new grammar.
     """
     new_quote = "'" if old_quote == '"' else '"'
-    parts = []
+    parts: list[str] = []
     previous_index = 0
     for start, end in iter_fexpr_spans(fstring):
-        parts.append(fstring[previous_index:start])
-        parts.append(fstring[start:end].replace(old_quote, new_quote))
+        parts.extend((
+            fstring[previous_index:start],
+            fstring[start:end].replace(old_quote, new_quote),
+        ))
         previous_index = end
     parts.append(fstring[previous_index:])
     return "".join(parts)
@@ -2257,10 +2259,7 @@ def do_transform(
         insert_str_child = insert_str_child_factory(LL[string_idx])
 
         comma_idx = -1
-        ends_with_comma = False
-        if LL[comma_idx].type == token.COMMA:
-            ends_with_comma = True
-
+        ends_with_comma = LL[comma_idx].type == token.COMMA
         leaves_to_steal_comments_from = [LL[string_idx]]
         if ends_with_comma:
             leaves_to_steal_comments_from.append(LL[comma_idx])

src/blib2to3/pgen2/driver.py

@@ -16,7 +16,6 @@
 __all__ = ["Driver", "load_grammar"]
 
 # Python imports
-import io
 import logging
 import os
 import pkgutil
@@ -37,7 +36,7 @@
 Path = Union[str, "os.PathLike[str]"]
 
 
-@dataclass
+@dataclass(slots=True, eq=False, repr=False)
 class ReleaseRange:
     start: int
     end: int | None = None
@@ -49,6 +48,8 @@ def lock(self) -> None:
 
 
 class TokenProxy:
+    __slots__ = ("_tokens", "_counter", "_release_ranges", "logger")
+
     def __init__(self, generator: Any) -> None:
         self._tokens = generator
         self._counter = 0

src/blib2to3/pgen2/parse.py

@@ -157,6 +157,17 @@ class Parser:
 
     """
 
+    __slots__ = (
+        "convert",
+        "grammar",
+        "is_backtracking",
+        "last_token",
+        "proxy",
+        "rootnode",
+        "stack",
+        "used_names",
+    )
+
     def __init__(self, grammar: Grammar, convert: Convert | None = None) -> None:
         """Constructor.
 
@@ -392,4 +403,3 @@ def pop(self) -> None:
                 node[-1].append(newnode)
             else:
                 self.rootnode = newnode
-                self.rootnode.used_names = self.used_names

src/blib2to3/pgen2/pgen.py

@@ -2,17 +2,17 @@
 # Licensed to PSF under a Contributor Agreement.
 
 import os
-from collections.abc import Iterator, Sequence
-from typing import IO, Any, NoReturn, Union
+from collections.abc import Iterator
+from typing import IO, Any, NoReturn
 
 from blib2to3.pgen2 import grammar, token, tokenize
 from blib2to3.pgen2.tokenize import TokenInfo
 
-Path = Union[str, "os.PathLike[str]"]
+Path = str | os.PathLike[str]
 
 
 class PgenGrammar(grammar.Grammar):
-    pass
+    __slots__ = ()
 
 
 class ParserGenerator:
@@ -37,8 +37,7 @@ def __init__(self, filename: Path, stream: IO[str] | None = None) -> None:
 
     def make_grammar(self) -> PgenGrammar:
         c = PgenGrammar()
-        names = list(self.dfas.keys())
-        names.sort()
+        names = sorted(self.dfas.keys())
         names.remove(self.startsymbol)
         names.insert(0, self.startsymbol)
         for name in names:
@@ -49,9 +48,10 @@ def make_grammar(self) -> PgenGrammar:
             dfa = self.dfas[name]
             states = []
             for state in dfa:
-                arcs = []
-                for label, next in sorted(state.arcs.items()):
-                    arcs.append((self.make_label(c, label), dfa.index(next)))
+                arcs: list[tuple[int, int]] = [
+                    (self.make_label(c, label), dfa.index(next_))
+                    for label, next_ in sorted(state.arcs.items())
+                ]
                 if state.isfinal:
                     arcs.append((0, dfa.index(state)))
                 states.append(arcs)
@@ -122,8 +122,7 @@ def make_label(self, c: PgenGrammar, label: str) -> int:
                     return ilabel
 
     def addfirstsets(self) -> None:
-        names = list(self.dfas.keys())
-        names.sort()
+        names = sorted(self.dfas.keys())
         for name in names:
             if name not in self.first:
                 self.calcfirst(name)
@@ -145,7 +144,7 @@ def calcfirst(self, name: str) -> None:
                     self.calcfirst(label)
                     fset = self.first[label]
                     assert fset is not None
-                totalset.update(fset)
+                totalset |= fset
                 overlapcheck[label] = fset
             else:
                 totalset[label] = 1
@@ -375,10 +374,7 @@ def __eq__(self, other: Any) -> bool:
         # would invoke this method recursively, with cycles...
         if len(self.arcs) != len(other.arcs):
             return False
-        for label, next in self.arcs.items():
-            if next is not other.arcs.get(label):
-                return False
-        return True
+        return all(next_ is other.arcs.get(label) for label, next_ in self.arcs.items())
 
     __hash__: Any = None  # For Py3 compatibility.