From a4531b8ff991add32c909315efc90bfba60b4549 Mon Sep 17 00:00:00 2001 From: Alpha Date: Fri, 24 Jul 2026 06:19:59 +0000 Subject: [PATCH 01/14] feat(transforms): implement all planned AST transforms Adds every transform listed in transforms/README.md that wasn't yet built (unfold IIFE lambdas, dummy-assignment/docstring/type-statement/local-import removal, typing-class/generic/overload/decorator/typing_extensions handling, NamedTuple/TypedDict-to-constructor conversion, dynamic attribute access conversion, early-exit/inline/lambda conversions, dead-block collapsing, and extended constant folding), and ports the two unregistered legacy free-function transforms (exception-bracket removal, positional-arg conversion) to the SuiteTransformer architecture. Adds a shared qualified-name resolution helper and the terser_hints marker package for opt-out docstring preservation. Also fixes three pre-existing bugs that were latent until these new subtree-deleting/import-rewriting transforms started exercising them: an always-false hasattr(arg, "ref") typo in the scope resolver that mis-scoped parameter annotations, a mangler crash on namespaces orphaned by deleted subtrees, and CombineImports unconditionally rebuilding (and silently double-binding) single un-combinable import statements. --- pyproject.toml | 2 +- src/terser/_pipeline/mangler/_locals.py | 9 +- src/terser/_pipeline/parser/_scope.py | 2 +- src/terser/_pipeline/transforms/__init__.py | 49 ++++++- .../transforms/cleanup_local_imports.py | 64 +++++++++ .../_pipeline/transforms/combine_imports.py | 56 ++++---- .../_pipeline/transforms/constant_folding.py | 114 ++++++++++++++- .../convert_dynamic_attribute_access.py | 66 +++++++++ .../transforms/convert_early_exits.py | 43 ++++++ .../_pipeline/transforms/convert_to_inline.py | 38 +++++ .../_pipeline/transforms/convert_to_lambda.py | 35 +++++ .../transforms/convert_typing_constructors.py | 136 ++++++++++++++++++ .../transforms/convert_typing_extensions.py | 35 +++++ src/terser/_pipeline/transforms/remove_all.py | 29 ++++ .../transforms/remove_dead_blocks.py | 40 ++++++ .../_pipeline/transforms/remove_docstrings.py | 61 ++++++++ .../transforms/remove_dummy_assignments.py | 36 +++++ .../transforms/remove_exception_brackets.py | 38 +++-- .../_pipeline/transforms/remove_generics.py | 31 ++++ .../transforms/remove_literal_statements.py | 13 +- .../_pipeline/transforms/remove_overloads.py | 37 +++++ .../_pipeline/transforms/remove_posargs.py | 33 +++-- .../transforms/remove_type_statements.py | 28 ++++ .../transforms/remove_typing_classes.py | 32 +++++ .../transforms/remove_typing_decorators.py | 36 +++++ .../_pipeline/transforms/unfold_iife.py | 32 +++++ src/terser/config.py | 59 ++++++++ src/terser/utils/imports.py | 37 +++++ src/terser_hints/__init__.py | 15 ++ 29 files changed, 1154 insertions(+), 52 deletions(-) create mode 100644 src/terser/_pipeline/transforms/cleanup_local_imports.py create mode 100644 src/terser/_pipeline/transforms/convert_dynamic_attribute_access.py create mode 100644 src/terser/_pipeline/transforms/convert_early_exits.py create mode 100644 src/terser/_pipeline/transforms/convert_to_inline.py create mode 100644 src/terser/_pipeline/transforms/convert_to_lambda.py create mode 100644 src/terser/_pipeline/transforms/convert_typing_constructors.py create mode 100644 src/terser/_pipeline/transforms/convert_typing_extensions.py create mode 100644 src/terser/_pipeline/transforms/remove_all.py create mode 100644 src/terser/_pipeline/transforms/remove_dead_blocks.py create mode 100644 src/terser/_pipeline/transforms/remove_docstrings.py create mode 100644 src/terser/_pipeline/transforms/remove_dummy_assignments.py create mode 100644 src/terser/_pipeline/transforms/remove_generics.py create mode 100644 src/terser/_pipeline/transforms/remove_overloads.py create mode 100644 src/terser/_pipeline/transforms/remove_type_statements.py create mode 100644 src/terser/_pipeline/transforms/remove_typing_classes.py create mode 100644 src/terser/_pipeline/transforms/remove_typing_decorators.py create mode 100644 src/terser/_pipeline/transforms/unfold_iife.py create mode 100644 src/terser/utils/imports.py create mode 100644 src/terser_hints/__init__.py diff --git a/pyproject.toml b/pyproject.toml index 4da99a6..997874b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["src/terser", "src/python_minifier", "src/alpha93"] +packages = ["src/terser", "src/python_minifier", "src/alpha93", "src/terser_hints"] [tool.ruff] src = ["src"] diff --git a/src/terser/_pipeline/mangler/_locals.py b/src/terser/_pipeline/mangler/_locals.py index e40d4bc..4718e93 100644 --- a/src/terser/_pipeline/mangler/_locals.py +++ b/src/terser/_pipeline/mangler/_locals.py @@ -99,7 +99,14 @@ def reserve_name(name, reservation_scope): """ for namespace in reservation_scope: - ref(namespace).assigned_names.add(name) + namespace_ref = ref(namespace) + if not hasattr(namespace_ref, 'assigned_names'): + # namespace is no longer reachable from the module root (a transform deleted + # the subtree it belonged to) - nothing will ever rename anything in it, so + # there's nothing to reserve + continue + + namespace_ref.assigned_names.add(name) def should_rename(binding, name, scope, is_available): diff --git a/src/terser/_pipeline/parser/_scope.py b/src/terser/_pipeline/parser/_scope.py index 0d2d073..a253f41 100644 --- a/src/terser/_pipeline/parser/_scope.py +++ b/src/terser/_pipeline/parser/_scope.py @@ -82,7 +82,7 @@ def __function_arguments(self, node: ast.arguments, fn: Invokable, /): for arg in (typed[list[ast.arg]].getattr(node, "posonlyargs", []) + node.args): self.__resolve(arg, namespace=fn) - if hasattr(arg, "ref") and arg.annotation is not None: + if arg.annotation is not None: self.__resolve(arg.annotation, namespace=namespace) if hasattr(node, "kwonlyargs"): diff --git a/src/terser/_pipeline/transforms/__init__.py b/src/terser/_pipeline/transforms/__init__.py index 26b7191..03a4aa7 100644 --- a/src/terser/_pipeline/transforms/__init__.py +++ b/src/terser/_pipeline/transforms/__init__.py @@ -9,19 +9,64 @@ from .remove_literal_statements import RemoveLiteralStatements from .remove_object_base import RemoveObject from .remove_pass import RemovePass +from .unfold_iife import UnfoldIIFE +from .remove_type_statements import RemoveTypeStatements +from .convert_typing_extensions import ConvertTypingExtensions +from .convert_early_exits import ConvertEarlyExits +from .convert_to_inline import ConvertToInline +from .convert_to_lambda import ConvertToLambda +from .remove_dummy_assignments import RemoveDummyAssignments +from .remove_docstrings import RemoveDocstrings +from .cleanup_local_imports import CleanupLocalImports +from .remove_overloads import RemoveOverloads +from .remove_typing_decorators import RemoveTypingDecorators +from .remove_generics import RemoveGenerics +from .remove_typing_classes import RemoveTypingClasses +from .convert_typing_constructors import ConvertTypingConstructors +from .convert_dynamic_attribute_access import ConvertDynamicAttributeAccess +from .remove_dead_blocks import RemoveDeadBlocks +from .remove_exception_brackets import RemoveExceptionBrackets +from .remove_posargs import RemovePosArgs +from .remove_all import RemoveAll __transforms__ = [ - Contracts, + # FLAGS = 0 (pre-resolve, pure syntax) + UnfoldIIFE, + RemoveTypeStatements, + ConvertTypingExtensions, RemoveLiteralStatements, CombineImports, - RemoveAnnotations, RemovePass, RemoveObject, RemoveAsserts, RemoveDebug, RemoveExplicitReturnNone, + ConvertEarlyExits, + ConvertToInline, + ConvertToLambda, + + # FLAGS = REQUIRES_IMPORT_RESOLVE + Contracts, + RemoveAnnotations, + RemoveDummyAssignments, + RemoveDocstrings, + CleanupLocalImports, + RemoveOverloads, + RemoveTypingDecorators, + RemoveGenerics, + RemoveTypingClasses, + ConvertTypingConstructors, + ConvertDynamicAttributeAccess, FoldConstants, + RemoveDeadBlocks, + + # FLAGS = REQUIRES_MODULE_RESOLVE + RemoveExceptionBrackets, + + # FLAGS = INFLUENCES_MANGLING + RemovePosArgs, + RemoveAll, ] __all__ = ("TransformCache", "__transforms__") diff --git a/src/terser/_pipeline/transforms/cleanup_local_imports.py b/src/terser/_pipeline/transforms/cleanup_local_imports.py new file mode 100644 index 0000000..5413f54 --- /dev/null +++ b/src/terser/_pipeline/transforms/cleanup_local_imports.py @@ -0,0 +1,64 @@ +from typing import override + +from terser.ast import ast, ref +from terser.config import TransformConfig +from ..resolver.binding import ImportBinding +from ._suite import SuiteTransformer, TransformerFlag + + +class CleanupLocalImports(SuiteTransformer): + """ + Remove unused local (function/class-scope) imports, and unused + module-level imports too if `config.respect_all` (and not exported) + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + _module: ast.Module + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.cleanup_local_imports + + @override + def visit_Module(self, node: ast.Module): + self._module = node + node.body = self.suite(node.body, parent=node) + return node + + def _clean_alias_list(self, stmt, in_module_scope: bool): + kept = [] + for alias in stmt.names: + if alias.name == '*': + return stmt.names # wildcard imports are left alone entirely + + binding = ref(alias).binding + if not isinstance(binding, ImportBinding): + kept.append(alias) + continue + + if in_module_scope and (not self._config.respect_all or binding.exported): + kept.append(alias) + continue + + if len(binding.references) > 1: + kept.append(alias) + + return kept + + @override + def suite(self, node_list, parent): + result = [] + for stmt in node_list: + if isinstance(stmt, (ast.Import, ast.ImportFrom)): + kept = self._clean_alias_list(stmt, in_module_scope=ref(stmt).namespace is self._module) + if not kept: + continue + stmt.names = kept + + result.append(self.visit(stmt)) + + if not result: + return [] if isinstance(parent, ast.Module) else [self.add_child(ast.Expr(ast.Num(0)), parent=parent)] + + return result diff --git a/src/terser/_pipeline/transforms/combine_imports.py b/src/terser/_pipeline/transforms/combine_imports.py index 0cfb3c8..fadb1a2 100644 --- a/src/terser/_pipeline/transforms/combine_imports.py +++ b/src/terser/_pipeline/transforms/combine_imports.py @@ -25,26 +25,31 @@ def suite(self, node_list, parent): def _combine_import(self, node_list, parent): - alias = [] - namespace = None + pending = [] + + def flush(): + if len(pending) > 1: + alias = [a for stmt in pending for a in stmt.names] + yield self.add_child(ast.Import(names=alias), parent=parent, namespace=None) + elif pending: + # nothing to combine - yield the original statement unchanged, rather than + # needlessly reconstructing (and re-binding) an identical Import node + yield pending[0] for statement in node_list: if isinstance(statement, ast.Import): - alias += statement.names + pending.append(statement) else: - if alias: - yield self.add_child(ast.Import(names=alias), parent=parent, namespace=namespace) - alias = [] + yield from flush() + pending = [] yield statement - if alias: - yield self.add_child(ast.Import(names=alias), parent=parent, namespace=namespace) + yield from flush() def _combine_import_from(self, node_list, parent): - prev_import = None - alias = [] + pending: list[ast.ImportFrom] = [] def combine(statement): if not isinstance(statement, ast.ImportFrom): @@ -53,28 +58,29 @@ def combine(statement): if len(statement.names) == 1 and statement.names[0].name == '*': return False - if prev_import is None: + if not pending: return True - if statement.module == prev_import.module and statement.level == prev_import.level: - return True + return statement.module == pending[0].module and statement.level == pending[0].level - return False + def flush(): + if len(pending) > 1: + alias = [a for stmt in pending for a in stmt.names] + yield self.add_child( + ast.ImportFrom(module=pending[0].module, names=alias, level=pending[0].level), parent=parent, namespace=ref(pending[0]).namespace + ) + elif pending: + # nothing to combine - yield the original statement unchanged, rather than + # needlessly reconstructing (and re-binding) an identical ImportFrom node + yield pending[0] for statement in node_list: if combine(statement): - prev_import = statement - alias += statement.names + pending.append(statement) else: - if alias: - yield self.add_child( - ast.ImportFrom(module=prev_import.module, names=alias, level=prev_import.level), parent=parent, namespace=ref(prev_import).namespace - ) - alias = [] + yield from flush() + pending = [] yield statement - if alias: - yield self.add_child( - ast.ImportFrom(module=prev_import.module, names=alias, level=prev_import.level), parent=parent, namespace=ref(prev_import).namespace - ) + yield from flush() diff --git a/src/terser/_pipeline/transforms/constant_folding.py b/src/terser/_pipeline/transforms/constant_folding.py index e607ed5..6d142e9 100644 --- a/src/terser/_pipeline/transforms/constant_folding.py +++ b/src/terser/_pipeline/transforms/constant_folding.py @@ -2,14 +2,27 @@ from typing import TYPE_CHECKING, override from terser.ast import ast, compare_ast, is_constant_node, ref +from terser.utils.imports import qualified_name +from ..resolver.binding import BuiltinBinding from ..printer.expression_printer import ExpressionPrinter -from ._suite import SuiteTransformer +from ._suite import SuiteTransformer, TransformerFlag if TYPE_CHECKING: from ...config import TransformConfig +def _is_unshadowed_builtin(node, name: str) -> bool: + if not isinstance(node, ast.Name): + return False + + binding = ref(node).binding + return isinstance(binding, BuiltinBinding) and binding.name == name and not binding.is_redefined() + + +_CONVERSION_FUNCS = {-1: 'str', 115: 'str', 114: 'repr', 97: 'ascii'} + + def is_foldable_constant(node): """ Check if a node is a constant expression that can participate in folding. @@ -34,7 +47,7 @@ class FoldConstants(SuiteTransformer): """ Fold Constants if it would reduce the size of the source """ - FLAGS = 0 + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE @override @classmethod @@ -133,6 +146,103 @@ def visit_UnaryOp(self, node): return self.fold(node) + def visit_Compare(self, node): + node.left = self.visit(node.left) + node.comparators = [self.visit(c) for c in node.comparators] + + if len(node.ops) != 1 or not isinstance(node.ops[0], (ast.Eq, ast.NotEq, ast.Is, ast.IsNot)): + return node + + left, right = node.left, node.comparators[0] + left_bool = is_constant_node(left, ast.NameConstant) and isinstance(left.value, bool) + right_bool = is_constant_node(right, ast.NameConstant) and isinstance(right.value, bool) + + if left_bool == right_bool: + # exactly one side must be a bool literal - both or neither isn't this pattern + return node + + bool_value, other = (left.value, right) if left_bool else (right.value, left) + negate = bool_value == isinstance(node.ops[0], (ast.NotEq, ast.IsNot)) + + new_node = ast.UnaryOp(op=ast.Not(), operand=other) if negate else other + node_ref = ref(node) + return self.add_child(new_node, node_ref.parent, node_ref.namespace) + + def visit_Name(self, node): + if node.id != '__debug__' or not _is_unshadowed_builtin(node, '__debug__'): + return node + + new_node = ast.NameConstant(value=self._config.optimize < 1) + node_ref = ref(node) + return self.add_child(new_node, node_ref.parent, node_ref.namespace) + + def visit_Attribute(self, node): + node.value = self.visit(node.value) + + if qualified_name(node) != 'typing.TYPE_CHECKING': + return node + + new_node = ast.NameConstant(value=False) + node_ref = ref(node) + return self.add_child(new_node, node_ref.parent, node_ref.namespace) + + def visit_JoinedStr(self, node): + node.values = [self.visit(v) for v in node.values] + + if any(isinstance(v, ast.FormattedValue) and v.format_spec is not None for v in node.values): + return node + + terms = [] + for v in node.values: + if isinstance(v, ast.Constant): + terms.append(v) + continue + + func_name = _CONVERSION_FUNCS.get(v.conversion, None) + if func_name is None: + return node + + terms.append(ast.Call(func=ast.Name(id=func_name, ctx=ast.Load()), args=[v.value], keywords=[])) + + if not terms: + new_node = ast.Constant(value='') + elif len(terms) == 1 and isinstance(terms[0], ast.Call): + new_node = terms[0] + else: + new_node = terms[0] + for term in terms[1:]: + new_node = ast.BinOp(left=new_node, op=ast.Add(), right=term) + + node_ref = ref(node) + return self.add_child(new_node, node_ref.parent, node_ref.namespace) + + def visit_Call(self, node): + node.func = self.visit(node.func) + node.args = [self.visit(a) for a in node.args] + node.keywords = [self.visit(k) for k in node.keywords] + + if node.keywords: + return node + + new_node = None + if not node.args and _is_unshadowed_builtin(node.func, 'list'): + new_node = ast.List(elts=[], ctx=ast.Load()) + elif not node.args and _is_unshadowed_builtin(node.func, 'dict'): + new_node = ast.Dict(keys=[], values=[]) + elif not node.args and _is_unshadowed_builtin(node.func, 'tuple'): + new_node = ast.Tuple(elts=[], ctx=ast.Load()) + elif ( + len(node.args) == 1 and isinstance(node.args[0], (ast.List, ast.Tuple)) and node.args[0].elts + and _is_unshadowed_builtin(node.func, 'set') + ): + new_node = ast.Set(elts=node.args[0].elts) + + if new_node is None: + return node + + node_ref = ref(node) + return self.add_child(new_node, node_ref.parent, node_ref.namespace) + def equal_value_and_type(a, b): if type(a) != type(b): diff --git a/src/terser/_pipeline/transforms/convert_dynamic_attribute_access.py b/src/terser/_pipeline/transforms/convert_dynamic_attribute_access.py new file mode 100644 index 0000000..051bb66 --- /dev/null +++ b/src/terser/_pipeline/transforms/convert_dynamic_attribute_access.py @@ -0,0 +1,66 @@ +import keyword +from typing import override + +from terser.ast import ast, ref +from terser.config import TransformConfig +from ..resolver.binding import BuiltinBinding +from ._suite import SuiteTransformer, TransformerFlag + + +def _valid_attr_name(node: ast.expr) -> str | None: + if not isinstance(node, ast.Constant) or not isinstance(node.value, str): + return None + + name = node.value + return name if name.isidentifier() and not keyword.iskeyword(name) else None + + +def _is_unshadowed_builtin(node: ast.expr, name: str) -> bool: + if not isinstance(node, ast.Name): + return False + + binding = ref(node).binding + return isinstance(binding, BuiltinBinding) and binding.name == name and not binding.is_redefined() + + +class ConvertDynamicAttributeAccess(SuiteTransformer): + """ + Convert `getattr(obj, "name")` to `obj.name`, and a bare + `setattr(obj, "name", value)` statement to `obj.name = value` + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.convert_dynamic_attribute_access + + @override + def visit_Call(self, node: ast.Call): + node: ast.Call = self.generic_visit(node) + + if node.keywords or len(node.args) != 2 or not _is_unshadowed_builtin(node.func, 'getattr'): + return node + + attr = _valid_attr_name(node.args[1]) + if attr is None: + return node + + new_node = ast.Attribute(value=node.args[0], attr=attr, ctx=ast.Load()) + return self.add_child(new_node, parent=ref(node).parent) + + @override + def visit_Expr(self, node: ast.Expr): + node.value = self.visit(node.value) + value = node.value + + if ( + isinstance(value, ast.Call) and not value.keywords and len(value.args) == 3 + and _is_unshadowed_builtin(value.func, 'setattr') + ): + attr = _valid_attr_name(value.args[1]) + if attr is not None: + new_node = ast.Assign(targets=[ast.Attribute(value=value.args[0], attr=attr, ctx=ast.Store())], value=value.args[2]) + return self.add_child(new_node, parent=ref(node).parent) + + return node diff --git a/src/terser/_pipeline/transforms/convert_early_exits.py b/src/terser/_pipeline/transforms/convert_early_exits.py new file mode 100644 index 0000000..ffa4cf6 --- /dev/null +++ b/src/terser/_pipeline/transforms/convert_early_exits.py @@ -0,0 +1,43 @@ +from typing import override + +from terser.ast import ast +from terser.config import TransformConfig +from ._suite import SuiteTransformer + + +class ConvertEarlyExits(SuiteTransformer): + """ + Merge `if cond: return a` immediately followed by `return b` into + `return a if cond else b` + """ + FLAGS = 0 + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.convert_early_exits + + @override + def suite(self, node_list, parent): + result = [] + i = 0 + while i < len(node_list): + node = node_list[i] + next_node = node_list[i + 1] if i + 1 < len(node_list) else None + + if ( + isinstance(node, ast.If) and not node.orelse + and len(node.body) == 1 and isinstance(node.body[0], ast.Return) + and isinstance(next_node, ast.Return) + ): + a = node.body[0].value or ast.Constant(value=None) + b = next_node.value or ast.Constant(value=None) + merged = ast.Return(value=ast.IfExp(test=node.test, body=a, orelse=b)) + result.append(self.add_child(merged, parent=parent)) + i += 2 + continue + + result.append(self.visit(node)) + i += 1 + + return result diff --git a/src/terser/_pipeline/transforms/convert_to_inline.py b/src/terser/_pipeline/transforms/convert_to_inline.py new file mode 100644 index 0000000..4c83d9e --- /dev/null +++ b/src/terser/_pipeline/transforms/convert_to_inline.py @@ -0,0 +1,38 @@ +from typing import override + +from terser.ast import ast, ref +from terser.config import TransformConfig +from ._suite import SuiteTransformer + + +class ConvertToInline(SuiteTransformer): + """ + Convert `if cond: func(x)` to `cond and func(x)`, and + `if fizz: foo()` / `else: bar()` to `foo() if fizz else bar()` + """ + FLAGS = 0 + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.convert_to_inline + + @override + def visit_If(self, node: ast.If): + node: ast.If = self.generic_visit(node) + + if len(node.body) != 1 or not isinstance(node.body[0], ast.Expr): + return node + + a = node.body[0].value + + if not node.orelse: + new_node = ast.Expr(value=ast.BoolOp(op=ast.And(), values=[node.test, a])) + return self.add_child(new_node, parent=ref(node).parent) + + if len(node.orelse) != 1 or not isinstance(node.orelse[0], ast.Expr): + return node + + b = node.orelse[0].value + new_node = ast.Expr(value=ast.IfExp(test=node.test, body=a, orelse=b)) + return self.add_child(new_node, parent=ref(node).parent) diff --git a/src/terser/_pipeline/transforms/convert_to_lambda.py b/src/terser/_pipeline/transforms/convert_to_lambda.py new file mode 100644 index 0000000..d8bae7f --- /dev/null +++ b/src/terser/_pipeline/transforms/convert_to_lambda.py @@ -0,0 +1,35 @@ +from typing import override + +from terser.ast import ast, ref +from terser.config import TransformConfig +from ._suite import SuiteTransformer + + +class ConvertToLambda(SuiteTransformer): + """ + Convert a single-expression function to a lambda assignment: + `def foo(...): return expr` -> `foo = lambda ...: expr` + """ + FLAGS = 0 + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.convert_to_lambda + + @override + def visit_FunctionDef(self, node: ast.FunctionDef): + node: ast.FunctionDef = self.generic_visit(node) + + if isinstance(node, ast.AsyncFunctionDef) or node.decorator_list or len(node.body) != 1: + return node + + stmt = node.body[0] + if not isinstance(stmt, ast.Return) or stmt.value is None: + # a bare `Expr` body isn't equivalent as a lambda: it always + # implicitly returns None, but a lambda returns the expression's value + return node + + body = stmt.value + new_node = ast.Assign(targets=[ast.Name(id=node.name, ctx=ast.Store())], value=ast.Lambda(args=node.args, body=body)) + return self.add_child(new_node, parent=ref(node).parent) diff --git a/src/terser/_pipeline/transforms/convert_typing_constructors.py b/src/terser/_pipeline/transforms/convert_typing_constructors.py new file mode 100644 index 0000000..56d69dc --- /dev/null +++ b/src/terser/_pipeline/transforms/convert_typing_constructors.py @@ -0,0 +1,136 @@ +from typing import override + +from terser.ast import ast, ref +from terser.config import TransformConfig +from terser.utils.imports import qualified_name +from ._suite import SuiteTransformer, TransformerFlag + +_NAMEDTUPLE_NAMES = ("typing.NamedTuple", "typing_extensions.NamedTuple") +_TYPEDDICT_NAMES = ("typing.TypedDict", "typing_extensions.TypedDict") + +# Sentinel: the class isn't safe to convert as-is - leave it as a plain ClassDef +_UNSAFE = object() + + +def _simple_fields(node: ast.ClassDef) -> list[ast.AnnAssign] | None: + if len(node.bases) != 1 or node.keywords: + return None + + fields: list[ast.AnnAssign] = [] + for stmt in node.body: + if not isinstance(stmt, ast.AnnAssign) or not isinstance(stmt.target, ast.Name): + return None + fields.append(stmt) + + return fields + + +class ConvertTypingConstructors(SuiteTransformer): + """ + Convert simple `NamedTuple`/`TypedDict` class definitions (fields only, no + methods) to `collections.namedtuple`/`dict` constructors, dropping the + dependency on `typing`. + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + _module: ast.Module + _collections_imported: bool + _needs_collections_import: bool + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.convert_typing_constructors + + @override + def visit_Module(self, node: ast.Module): + self._module = node + self._collections_imported = any( + isinstance(stmt, ast.Import) and any(a.name == 'collections' for a in stmt.names) for stmt in node.body + ) + self._needs_collections_import = False + + node.body = self.suite(node.body, parent=node) + + if self._needs_collections_import and not self._collections_imported: + node.body.insert(0, self.add_child(ast.Import(names=[ast.alias(name='collections', asname=None)]), parent=node)) + + return node + + def _ensure_collections_import(self): + if not self._collections_imported: + self._needs_collections_import = True + + def _convert_named_tuple(self, node: ast.ClassDef, fields: list[ast.AnnAssign]): + names: list[str] = [] + for f in fields: + assert isinstance(f.target, ast.Name) + names.append(f.target.id) + + values = [f.value for f in fields] + + # namedtuple only allows a trailing run of fields to have defaults + first_default = next((i for i, v in enumerate(values) if v is not None), len(values)) + if any(v is None for v in values[first_default:]): + return _UNSAFE + + defaults: list[ast.expr] = [] + for v in values[first_default:]: + assert v is not None + defaults.append(v) + + call = ast.Call( + func=ast.Attribute(value=ast.Name(id='collections', ctx=ast.Load()), attr='namedtuple', ctx=ast.Load()), + args=[ast.Constant(value=node.name), ast.Tuple(elts=[ast.Constant(value=n) for n in names], ctx=ast.Load())], + keywords=[ast.keyword(arg='defaults', value=ast.Tuple(elts=defaults, ctx=ast.Load()))] if defaults else [], + ) + new_node = ast.Assign(targets=[ast.Name(id=node.name, ctx=ast.Store())], value=call) + + self._ensure_collections_import() + return self.add_child(new_node, parent=ref(node).parent) + + def _convert_typed_dict(self, node: ast.ClassDef): + binding = ref(node).binding + other_refs = [r for r in binding.references if r is not node] + + for r in other_refs: + if not isinstance(r, ast.Name): + return _UNSAFE # used somewhere other than a plain name (e.g. as a base class) + + call = ref(r).parent + if not isinstance(call, ast.Call) or call.func is not r or call.args: + return _UNSAFE # not a pure-keyword constructor call + + for r in other_refs: + call = ref(r).parent + assert isinstance(call, ast.Call) + call.func = self.add_child(ast.Name(id='dict', ctx=ast.Load()), parent=call) + + return None # the class definition itself is dropped + + @override + def suite(self, node_list, parent): + result = [] + for node in node_list: + fields = _simple_fields(node) if isinstance(node, ast.ClassDef) else None + base_name = qualified_name(node.bases[0]) if fields is not None else None + + if fields is not None and base_name in _NAMEDTUPLE_NAMES: + converted = self._convert_named_tuple(node, fields) + result.append(self.visit(node) if converted is _UNSAFE else converted) + continue + + if fields is not None and base_name in _TYPEDDICT_NAMES: + converted = self._convert_typed_dict(node) + if converted is _UNSAFE: + result.append(self.visit(node)) + elif converted is not None: + result.append(self.visit(converted)) + continue + + result.append(self.visit(node)) + + if not result: + return [] if isinstance(parent, ast.Module) else [self.add_child(ast.Expr(ast.Num(0)), parent=parent)] + + return result diff --git a/src/terser/_pipeline/transforms/convert_typing_extensions.py b/src/terser/_pipeline/transforms/convert_typing_extensions.py new file mode 100644 index 0000000..2dff6e3 --- /dev/null +++ b/src/terser/_pipeline/transforms/convert_typing_extensions.py @@ -0,0 +1,35 @@ +from typing import override + +from terser.ast import ast +from terser.config import TransformConfig +from ._suite import SuiteTransformer + +# Symbols long-stable in `typing` (project requires Python >=3.14) - safe to +# rewrite a `from typing_extensions import X` to `from typing import X` +_STABLE_IN_TYPING = frozenset(( + "Protocol", "TypedDict", "Literal", "Final", "overload", "override", "TypeAlias", "ParamSpec", + "Concatenate", "Self", "Never", "assert_never", "assert_type", "runtime_checkable", "get_args", "get_origin", +)) + + +class ConvertTypingExtensions(SuiteTransformer): + """ + Convert `from typing_extensions import X` to `from typing import X`, for + symbols that are stable in `typing`. Unknown/newer `typing_extensions`-only + names (or a mix of stable and unknown names in one statement) are left + untouched. + """ + FLAGS = 0 + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.convert_typing_extensions + + @override + def visit_ImportFrom(self, node: ast.ImportFrom): + if node.module != "typing_extensions" or any(a.name not in _STABLE_IN_TYPING for a in node.names): + return node + + node.module = "typing" + return node diff --git a/src/terser/_pipeline/transforms/remove_all.py b/src/terser/_pipeline/transforms/remove_all.py new file mode 100644 index 0000000..f67cd64 --- /dev/null +++ b/src/terser/_pipeline/transforms/remove_all.py @@ -0,0 +1,29 @@ +from typing import override + +from terser.ast import ast +from terser.config import TransformConfig +from ._suite import SuiteTransformer, TransformerFlag + + +def _is_dunder_all_assign(node) -> bool: + return ( + isinstance(node, ast.Assign) and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) and node.targets[0].id == '__all__' + ) + + +class RemoveAll(SuiteTransformer): + """ + Remove the top-level `__all__` assignment + """ + FLAGS = TransformerFlag.INFLUENCES_MANGLING + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.remove_dunder_all + + @override + def visit_Module(self, node: ast.Module): + node.body = [stmt for stmt in node.body if not _is_dunder_all_assign(stmt)] + return node diff --git a/src/terser/_pipeline/transforms/remove_dead_blocks.py b/src/terser/_pipeline/transforms/remove_dead_blocks.py new file mode 100644 index 0000000..ecdcf2e --- /dev/null +++ b/src/terser/_pipeline/transforms/remove_dead_blocks.py @@ -0,0 +1,40 @@ +from typing import override + +from terser.ast import ast, ref +from terser.config import TransformConfig +from ._suite import SuiteTransformer, TransformerFlag + + +class RemoveDeadBlocks(SuiteTransformer): + """ + Collapse `if` statements whose test is a literal constant bool: + `if True: body` -> `body`, `if False: body` (else: `orelse`) -> `orelse` + + Registered directly after `FoldConstants` (same FLAGS stage), which is + what folds comparisons/`__debug__`/`typing.TYPE_CHECKING` down to a literal + bool test in the first place. + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.remove_debug or config.fold_constants + + @override + def suite(self, node_list, parent): + result = [] + for node in node_list: + if isinstance(node, ast.If) and isinstance(node.test, ast.Constant) and isinstance(node.test.value, bool): + branch = node.body if node.test.value else node.orelse + for stmt in branch: + ref(stmt).parent = parent + result.extend(self.suite(branch, parent)) + continue + + result.append(self.visit(node)) + + if not result: + return [] if isinstance(parent, ast.Module) else [self.add_child(ast.Expr(ast.Num(0)), parent=parent)] + + return result diff --git a/src/terser/_pipeline/transforms/remove_docstrings.py b/src/terser/_pipeline/transforms/remove_docstrings.py new file mode 100644 index 0000000..53e623c --- /dev/null +++ b/src/terser/_pipeline/transforms/remove_docstrings.py @@ -0,0 +1,61 @@ +from typing import override + +from terser.ast import ast, is_constant_node +from terser.config import RemoveDocstringOptions, TransformConfig +from terser.utils.imports import qualified_name +from ._suite import SuiteTransformer, TransformerFlag + + +class RemoveDocstrings(SuiteTransformer): + """ + Remove docstrings, preserving module docstrings unless `also_modules` is + set, and preserving anything decorated with `@terser_hints.preserve_docstring` + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + def __init__(self, ctx, /): + super().__init__(ctx) + self._options = RemoveDocstringOptions() if isinstance(self._config.remove_docstrings, bool) else self._config.remove_docstrings + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.remove_docstrings is not False + + def _has_docstring(self, body) -> bool: + return bool(body) and isinstance(body[0], ast.Expr) and is_constant_node(body[0].value, ast.Str) + + def _preserved(self, decorator_list) -> bool: + return any(qualified_name(d) == "terser_hints.preserve_docstring" for d in decorator_list) + + def _strip_docstring(self, node): + node.body = node.body[1:] + if not node.body: + node.body = [self.add_child(ast.Expr(ast.Num(0)), parent=node)] + + @override + def visit_Module(self, node: ast.Module): + node.body = self.suite(node.body, parent=node) + + if self._options.also_modules and self._has_docstring(node.body): + node.body = node.body[1:] + + return node + + @override + def visit_ClassDef(self, node: ast.ClassDef): + node: ast.ClassDef = super().visit_ClassDef(node) + + if self._has_docstring(node.body) and not self._preserved(node.decorator_list): + self._strip_docstring(node) + + return node + + @override + def visit_FunctionDef(self, node: ast.FunctionDef): + node: ast.FunctionDef = super().visit_FunctionDef(node) + + if self._has_docstring(node.body) and not self._preserved(node.decorator_list): + self._strip_docstring(node) + + return node diff --git a/src/terser/_pipeline/transforms/remove_dummy_assignments.py b/src/terser/_pipeline/transforms/remove_dummy_assignments.py new file mode 100644 index 0000000..1b49c90 --- /dev/null +++ b/src/terser/_pipeline/transforms/remove_dummy_assignments.py @@ -0,0 +1,36 @@ +from typing import override + +from terser.ast import ast, ref +from terser.config import TransformConfig +from ._suite import SuiteTransformer, TransformerFlag + + +class RemoveDummyAssignments(SuiteTransformer): + """ + Remove self-assignments like `x = x` + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.remove_dummy_assignments + + @override + def suite(self, node_list, parent): + result = [self.visit(node) for node in node_list if not self._is_dummy(node)] + + if not result: + return [] if isinstance(parent, ast.Module) else [self.add_child(ast.Expr(ast.Num(0)), parent=parent)] + + return result + + def _is_dummy(self, node) -> bool: + if not isinstance(node, ast.Assign) or len(node.targets) != 1: + return False + + target, value = node.targets[0], node.value + if not isinstance(target, ast.Name) or not isinstance(value, ast.Name): + return False + + return ref(target).binding is ref(value).binding diff --git a/src/terser/_pipeline/transforms/remove_exception_brackets.py b/src/terser/_pipeline/transforms/remove_exception_brackets.py index 5031131..ad40c1b 100644 --- a/src/terser/_pipeline/transforms/remove_exception_brackets.py +++ b/src/terser/_pipeline/transforms/remove_exception_brackets.py @@ -8,8 +8,14 @@ We can't generally know if a name refers to an exception, so we only do this for builtin exceptions """ +from typing import TYPE_CHECKING, override + from terser.ast import ast, ref from ..resolver.binding import BuiltinBinding +from ._suite import SuiteTransformer, TransformerFlag + +if TYPE_CHECKING: + from terser.config import TransformConfig # These are always exceptions, in every version of python @@ -101,18 +107,28 @@ def _remove_empty_call(binding: BuiltinBinding): ref(name_node).parent = raise_node -def remove_no_arg_exception_call(module): - assert isinstance(module, ast.Module) +class RemoveExceptionBrackets(SuiteTransformer): + """ + Remove brackets with empty arguments from built-in exception raise statements + """ + FLAGS = TransformerFlag.REQUIRES_MODULE_RESOLVE - for binding in module.bindings: - if not isinstance(binding, BuiltinBinding): - continue + @override + @classmethod + def is_enabled(cls, config: "TransformConfig", /) -> bool: + return config.remove_empty_exc_brackets - if binding.is_redefined(): - continue + @override + def visit_Module(self, node): + for binding in ref(node).bindings: + if not isinstance(binding, BuiltinBinding): + continue + + if binding.is_redefined(): + continue - if binding.name in builtin_exceptions: - # We can remove any calls to builtin exceptions - _remove_empty_call(binding) + if binding.name in builtin_exceptions: + # We can remove any calls to builtin exceptions + _remove_empty_call(binding) - return module + return node diff --git a/src/terser/_pipeline/transforms/remove_generics.py b/src/terser/_pipeline/transforms/remove_generics.py new file mode 100644 index 0000000..07f1350 --- /dev/null +++ b/src/terser/_pipeline/transforms/remove_generics.py @@ -0,0 +1,31 @@ +from typing import override + +from terser.ast import ast +from terser.config import TransformConfig +from terser.utils.imports import qualified_name +from ._suite import SuiteTransformer, TransformerFlag + + +def _is_bare_generic(base: ast.expr) -> bool: + return isinstance(base, (ast.Name, ast.Attribute)) and qualified_name(base) == "typing.Generic" + + +class RemoveGenerics(SuiteTransformer): + """ + Remove bare (non-parametrized) `Generic` base classes. + + `Generic[T]` is left alone - the subscript form has real runtime behavior + (`__class_getitem__`) that a bare `Generic` base doesn't add. + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.remove_generics + + @override + def visit_ClassDef(self, node: ast.ClassDef): + node: ast.ClassDef = super().visit_ClassDef(node) + node.bases = [b for b in node.bases if not _is_bare_generic(b)] + return node diff --git a/src/terser/_pipeline/transforms/remove_literal_statements.py b/src/terser/_pipeline/transforms/remove_literal_statements.py index a4032d4..d66be05 100644 --- a/src/terser/_pipeline/transforms/remove_literal_statements.py +++ b/src/terser/_pipeline/transforms/remove_literal_statements.py @@ -51,8 +51,19 @@ def is_literal_statement(self, node): return is_constant_node(node.value, (ast.Num, ast.Str, ast.NameConstant, ast.Bytes)) + def _is_docstring_position(self, node_list, index, parent): + # leave docstrings alone here - RemoveDocstrings decides whether to remove them + if index != 0 or not isinstance(parent, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + return False + + node = node_list[0] + return isinstance(node, ast.Expr) and is_constant_node(node.value, ast.Str) + def suite(self, node_list, parent): - without_literals = [self.visit(n) for n in node_list if not self.is_literal_statement(n)] + without_literals = [ + self.visit(n) for i, n in enumerate(node_list) + if self._is_docstring_position(node_list, i, parent) or not self.is_literal_statement(n) + ] if len(without_literals) == 0: if isinstance(parent, ast.Module): diff --git a/src/terser/_pipeline/transforms/remove_overloads.py b/src/terser/_pipeline/transforms/remove_overloads.py new file mode 100644 index 0000000..8ebca6d --- /dev/null +++ b/src/terser/_pipeline/transforms/remove_overloads.py @@ -0,0 +1,37 @@ +from typing import override + +from terser.ast import ast +from terser.config import TransformConfig +from terser.utils.imports import qualified_name +from ._suite import SuiteTransformer, TransformerFlag + +_OVERLOAD_NAMES = ("typing.overload", "typing_extensions.overload") + + +def _is_overload(node) -> bool: + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return False + + return any(qualified_name(d) in _OVERLOAD_NAMES for d in node.decorator_list) + + +class RemoveOverloads(SuiteTransformer): + """ + Remove `@typing.overload`-decorated stub definitions - only the final, + undecorated implementation remains + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.remove_overloads or config.remove_typing_decorators + + @override + def suite(self, node_list, parent): + result = [self.visit(node) for node in node_list if not _is_overload(node)] + + if not result: + return [] if isinstance(parent, ast.Module) else [self.add_child(ast.Expr(ast.Num(0)), parent=parent)] + + return result diff --git a/src/terser/_pipeline/transforms/remove_posargs.py b/src/terser/_pipeline/transforms/remove_posargs.py index 86dea23..b80e69c 100644 --- a/src/terser/_pipeline/transforms/remove_posargs.py +++ b/src/terser/_pipeline/transforms/remove_posargs.py @@ -1,12 +1,29 @@ -import terser.ast.ast as ast +from typing import TYPE_CHECKING, override +from terser.ast import ast +from ._suite import SuiteTransformer, TransformerFlag -def remove_posargs(node): - if isinstance(node, ast.arguments) and hasattr(node, 'posonlyargs'): - node.args = node.posonlyargs + node.args - node.posonlyargs = [] +if TYPE_CHECKING: + from terser.config import TransformConfig - for child in ast.iter_child_nodes(node): - remove_posargs(child) - return node +class RemovePosArgs(SuiteTransformer): + """ + Convert positional-only arguments to normal arguments + """ + FLAGS = TransformerFlag.INFLUENCES_MANGLING + + @override + @classmethod + def is_enabled(cls, config: "TransformConfig", /) -> bool: + return config.convert_posargs + + @override + def visit_arguments(self, node: ast.arguments): + node: ast.arguments = self.generic_visit(node) + + if hasattr(node, 'posonlyargs') and node.posonlyargs: + node.args = node.posonlyargs + node.args + node.posonlyargs = [] + + return node diff --git a/src/terser/_pipeline/transforms/remove_type_statements.py b/src/terser/_pipeline/transforms/remove_type_statements.py new file mode 100644 index 0000000..d3f5408 --- /dev/null +++ b/src/terser/_pipeline/transforms/remove_type_statements.py @@ -0,0 +1,28 @@ +from typing import override + +from terser.ast import ast +from terser.config import TransformConfig +from ._suite import SuiteTransformer + +_TypeAlias = getattr(ast, "TypeAlias", ()) + + +class RemoveTypeStatements(SuiteTransformer): + """ + Remove `type X = ...` alias statements + """ + FLAGS = 0 + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.remove_type_statements and bool(_TypeAlias) + + @override + def suite(self, node_list, parent): + result = [self.visit(node) for node in node_list if not isinstance(node, _TypeAlias)] + + if not result: + return [] if isinstance(parent, ast.Module) else [self.add_child(ast.Expr(ast.Num(0)), parent=parent)] + + return result diff --git a/src/terser/_pipeline/transforms/remove_typing_classes.py b/src/terser/_pipeline/transforms/remove_typing_classes.py new file mode 100644 index 0000000..9dcb7e7 --- /dev/null +++ b/src/terser/_pipeline/transforms/remove_typing_classes.py @@ -0,0 +1,32 @@ +from typing import override + +from terser.ast import ast +from terser.config import TransformConfig +from terser.utils.imports import qualified_name +from ._suite import SuiteTransformer, TransformerFlag + +_PROTOCOL_NAMES = ("typing.Protocol", "typing_extensions.Protocol") +_RUNTIME_CHECKABLE_NAMES = ("typing.runtime_checkable", "typing_extensions.runtime_checkable") + + +class RemoveTypingClasses(SuiteTransformer): + """ + Remove bare `Protocol` base classes, unless the class is decorated with + `@typing.runtime_checkable` (removing `Protocol` there would break `isinstance`) + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.remove_typing_classes + + @override + def visit_ClassDef(self, node: ast.ClassDef): + node: ast.ClassDef = super().visit_ClassDef(node) + + if any(qualified_name(d) in _RUNTIME_CHECKABLE_NAMES for d in node.decorator_list): + return node + + node.bases = [b for b in node.bases if qualified_name(b) not in _PROTOCOL_NAMES] + return node diff --git a/src/terser/_pipeline/transforms/remove_typing_decorators.py b/src/terser/_pipeline/transforms/remove_typing_decorators.py new file mode 100644 index 0000000..2e0cc29 --- /dev/null +++ b/src/terser/_pipeline/transforms/remove_typing_decorators.py @@ -0,0 +1,36 @@ +from typing import override + +from terser.ast import ast +from terser.config import TransformConfig +from terser.utils.imports import qualified_name +from ._suite import SuiteTransformer, TransformerFlag + +_REMOVABLE_NAMES = ("typing.override", "typing_extensions.override", "typing.final", "typing_extensions.final") + + +def _strip(decorator_list: list[ast.expr]) -> list[ast.expr]: + return [d for d in decorator_list if qualified_name(d) not in _REMOVABLE_NAMES] + + +class RemoveTypingDecorators(SuiteTransformer): + """ + Remove `@typing.override`/`@typing.final` decorators + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.remove_typing_decorators + + @override + def visit_ClassDef(self, node: ast.ClassDef): + node: ast.ClassDef = super().visit_ClassDef(node) + node.decorator_list = _strip(node.decorator_list) + return node + + @override + def visit_FunctionDef(self, node: ast.FunctionDef): + node: ast.FunctionDef = super().visit_FunctionDef(node) + node.decorator_list = _strip(node.decorator_list) + return node diff --git a/src/terser/_pipeline/transforms/unfold_iife.py b/src/terser/_pipeline/transforms/unfold_iife.py new file mode 100644 index 0000000..f645153 --- /dev/null +++ b/src/terser/_pipeline/transforms/unfold_iife.py @@ -0,0 +1,32 @@ +from typing import override + +from terser.ast import ast, ref +from terser.config import TransformConfig +from ._suite import SuiteTransformer + + +class UnfoldIIFE(SuiteTransformer): + """ + Inline immediately-invoked no-arg lambda calls: `(lambda: x)()` -> `x` + """ + FLAGS = 0 + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.unfold_iife_lambdas + + @override + def visit_Call(self, node: ast.Call): + node: ast.Call = self.generic_visit(node) + + if node.args or node.keywords or not isinstance(node.func, ast.Lambda): + return node + + args = node.func.args + if args.posonlyargs or args.args or args.kwonlyargs or args.vararg or args.kwarg: + return node + + body = node.func.body + ref(body).parent = ref(node).parent + return body diff --git a/src/terser/config.py b/src/terser/config.py index 4b844c7..2bd49cb 100644 --- a/src/terser/config.py +++ b/src/terser/config.py @@ -19,6 +19,14 @@ class RemoveAnnotationOptions: """Remove class attribute annotations""" +@dataclass() +class RemoveDocstringOptions: + """Options that affect how docstrings are removed""" + + also_modules: bool = False + """Also remove module-level docstrings""" + + @dataclass() class TransformConfig: passes: int = 5 @@ -58,10 +66,61 @@ class TransformConfig: convert_pass: bool = True """Remove or convert `pass` statements to the smallest literal statement, like `0`""" + unfold_iife_lambdas: bool = True + """Inline immediately-invoked no-arg lambda calls, e.g. `(lambda: x)()` -> `x`""" + + remove_type_statements: bool = True + """Remove `type X = ...` alias statements""" + + convert_early_exits: bool = True + """Merge `if cond: return a` followed by `return b` into `return a if cond else b`""" + + convert_to_inline: bool = True + """Convert `if cond: func(x)` to `cond and func(x)`, and if/else statements to a conditional expression""" + + convert_to_lambda: bool = True + """Convert single-expression functions to a lambda assignment""" + ### requires binding + remove_dummy_assignments: bool = True + """Remove self-assignments like `x = x`""" + + remove_docstrings: bool | RemoveDocstringOptions = False + """Options that affect how docstrings are removed""" + + respect_all: bool = False + """When cleaning up unused imports, also remove unused module-level imports not listed in `__all__`""" + + cleanup_local_imports: bool = True + """Remove unused local imports, and unused global imports if `respect_all`""" + + remove_typing_decorators: bool = True + """Remove `@typing.override`/`@typing.final` decorators""" + + remove_overloads: bool = True + """Remove `@typing.overload`-decorated stub definitions. Always on when `remove_typing_decorators` is set""" + + remove_generics: bool = True + """Remove bare (non-parametrized) `Generic` base classes""" + + remove_typing_classes: bool = True + """Remove bare `Protocol` base classes (unless `@typing.runtime_checkable`)""" + + convert_typing_constructors: bool = True + """Convert simple `NamedTuple`/`TypedDict` class definitions to `namedtuple`/`dict` constructors""" + + convert_typing_extensions: bool = True + """Convert `typing_extensions` imports to `typing` where the symbol is stable there""" + + convert_dynamic_attribute_access: bool = True + """Convert `getattr`/`setattr` calls with a constant, valid identifier name to attribute access""" + remove_empty_exc_brackets: bool = True """Remove brackets with empty arguments from built-in exception raise statements""" ### mangle-sensitive transforms convert_posargs: bool = True """Convert positional-only arguments to normal arguments""" + + remove_dunder_all: bool = True + """Remove the top-level `__all__` assignment""" diff --git a/src/terser/utils/imports.py b/src/terser/utils/imports.py new file mode 100644 index 0000000..5160cd2 --- /dev/null +++ b/src/terser/utils/imports.py @@ -0,0 +1,37 @@ +from terser.ast import ast, ref +from .._pipeline.resolver.binding import ImportBinding + + +def qualified_name(node: ast.expr, /) -> str | None: + """ + Resolve a `Name` or dotted `Attribute` expression to the dotted path of the + import it refers to (e.g. `cast` after `from typing import cast` or + `typing.cast` both resolve to `"typing.cast"`), using the already-computed + name bindings. Returns None if the root name isn't an import, or its + binding has no name. + """ + + attrs: list[str] = [] + while isinstance(node, ast.Attribute): + attrs.append(node.attr) + node = node.value + attrs.reverse() + + if not isinstance(node, ast.Name): + return None + + binding = ref(node).binding + if not isinstance(binding, ImportBinding) or not binding.name: + return None + + source_module = binding.source_module + if not source_module: + return None + + # `attrs` non-empty means this was a dotted access off an `import x` binding + # (e.g. `typing.cast`) - the module identity comes from source_module, not + # the (possibly aliased) local name. With no attrs, this is a bare `Name` + # from a `from x import y` binding, where binding.name is the local + # (possibly aliased) symbol name. + tail = ".".join(attrs) if attrs else binding.name + return f"{source_module}.{tail}" diff --git a/src/terser_hints/__init__.py b/src/terser_hints/__init__.py new file mode 100644 index 0000000..5472b64 --- /dev/null +++ b/src/terser_hints/__init__.py @@ -0,0 +1,15 @@ +from typing import TypeVar + +_T = TypeVar("_T") + + +def preserve_docstring(obj: _T, /) -> _T: + """ + Marker decorator: tells terser to keep this class/function's docstring + even when docstring removal is otherwise enabled. Detected statically, + a no-op at runtime. + """ + return obj + + +__all__ = ("preserve_docstring",) From 634cbbf010bf849d81b4717cf470e4890c11b2f3 Mon Sep 17 00:00:00 2001 From: Alpha Date: Fri, 24 Jul 2026 06:35:23 +0000 Subject: [PATCH 02/14] fix(transforms): make config.passes actually iterate to a fixed point TransformCache.passes/.transforms were declared but never populated, so the "stop when nothing changed" break after each pass-loop was always comparing an empty dict - every multi-pass loop (single-file module transforms, project-wide transforms, and the mangle-sensitive tail pass) silently ran exactly once regardless of config.passes. Add transforms.apply_pass(), which applies one ordered stage of enabled transforms and records per-transform whether it changed the module (via an ast.dump diff) into cache.passes - the exact signal SuiteTransformer.__new__ was already built to consume for its skip-unchanged-upstream optimization. Wire it into the three loop sites in _minify.py, project.py, and terser.py. project.py additionally now gives each collected module its own TransformCache, since the shared one it reused before would have conflated unrelated modules' change state once the tracking actually started working. Also document the actual behavior of every implemented transform in transforms/README.md (previously just a plan; the constant-folding entry notes which sub-items - numeric literal reformatting, string unescaping, sys.version_info/platform folding - aren't implemented and why). --- src/terser/_minify.py | 6 +- src/terser/_pipeline/transforms/README.md | 95 +++++++++----------- src/terser/_pipeline/transforms/__init__.py | 4 +- src/terser/_pipeline/transforms/__init__.pyi | 4 +- src/terser/_pipeline/transforms/_suite.py | 21 +++++ src/terser/project.py | 18 ++-- src/terser/terser.py | 8 +- 7 files changed, 82 insertions(+), 74 deletions(-) diff --git a/src/terser/_minify.py b/src/terser/_minify.py index b7890ba..7526d15 100644 --- a/src/terser/_minify.py +++ b/src/terser/_minify.py @@ -79,11 +79,7 @@ def minify( cache = transforms.TransformCache(config) for _ in task.range(config.passes, "Applying transforms"): - for transform in transforms.__transforms__: - if not transform.is_enabled(config) or transform.FLAGS > 1: - continue - - module: ast.Module = transform(cache)(module) + module = transforms.apply_pass(cache, module, transforms.__transforms__, 1) if not any(cache.passes.values()): break diff --git a/src/terser/_pipeline/transforms/README.md b/src/terser/_pipeline/transforms/README.md index a35d116..a0d1e87 100644 --- a/src/terser/_pipeline/transforms/README.md +++ b/src/terser/_pipeline/transforms/README.md @@ -1,57 +1,48 @@ -## Planned transforms +## Implemented transforms -- Contracts (`contracts.py`) `(Flags.REQUIRES_IMPORT_RESOLVE)` +All entries below are implemented and registered in `__transforms__` (`__init__.py`). Config field names are on `TransformConfig` (`terser/config.py`) unless noted. + +- Contracts (`contracts.py`, `Contracts`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.apply_contracts`, rules in `config.contracts`. - Default contracts: `typing.cast(_, value) -> value` (leave only `value`), `typing.assert_never(x) -> None` (completely remove call), `typing.assert_type(x, _) -> x` (leave only `x`). -- Unfold `@lambda _: _()` constants (inline function body) -- Remove dummy assignments `(Flags.REQUIRES_IMPORT_RESOLVE)` -- Remove literal statements (`remove_literal_statements.py`) — SHOULD preserve docstrings, will process them right next. -- Remove docstrings `(param: also_modules)` — preserve docstrings in modules if `not also_modules`. preserve docstrings if decorated with `@terser_hints.preserve_docstring` (can be changed later). -- Combine imports `combine_imports.py` -- Cleanup local imports `(Flags.REQUIRES_IMPORT_RESOLVE)` — remove unused local imports, and global imports if `config.respect_all`. -- Remove annotations `remove_annotations.py` -- Remove `type` statements -- Remove typing classes `(Flags.REQUIRES_IMPORT_RESOLVE)` - - bare `Protocol` (SHOULD ignore if decorated with `@typing.runtime_visible`) - - `NamedTuple`, `NamedDict` (SHOULD convert constructors into `tuple`, `dict`) -- Remove `Generic`s `(Flags.REQUIRES_IMPORT_RESOLVE)` -- Remove `@overload`s `(Flags.REQUIRES_IMPORT_RESOLVE)` - - Always `True` if `remove-typing-decorators` is selected -- Remove typing decorators `(Flags.REQUIRES_IMPORT_RESOLVE)` - - `@typing.override` - - `@typing.final` -- Remove explicit `return None` `remove_explicit_return_none.py` -- Remove explicit trailing `return` -- Fold constants `constant_folding.py` - - constant operations - - boolean operations (`x ==/is True` → `x`, `x ==/is False` → `not x`, …) - - `__debug__` - - `typing.TYPE_CHECKING` `(typing)` - - `sys.version_info`, `sys.platform` `(module-sensitive)` - - numbers (`0b1` → `1`, `1_000_000` → `1e6`, `0.0001` → `1e-4`) - - strings (`\uXXXX` → raw represents, `f"{x}"` → `str(x)` or `x`, `f"{x}{y}"` → `x + y`) - - collections ( `list()` → `[]`, `dict()` → `{}`, `tuple()` → `()`, `set([1])` → `{1,}` ) -- Convert `typing_extensions` `(Flags.REQUIRES_IMPORT_RESOLVE)` -- Remove dead blocks -- Convert early exists -- Convert to inline - - `if cond: func(x)` → `cond and func(x)` - - `if fizz: foo(); else: bar()` → `foo() if fizz else bar()` -- Convert to lambda - - `def foo(...): single_expr()` → `foo = lambda ...: single_expr()` -- Convert dynamic attribute access `(Flags.REQUIRES_IMPORT_RESOLVE)` - - `getattr(obj, name)` → `obj.name`, `setattr(obj, name, value)` → `obj.name = value` - - SHOULD ignore when: - - `name` is not constant - - `name` breaks Python naming requirements - - `getattr` has default value -- Remove unnecessary base/meta classes `remove_object_base.py` `(Flags.REQUIRES_MODULE_RESOLVE)` - - Default remove: `object` -- Remove empty exception brackets `remove_exception_brackets.py` `(Flags.REQUIRES_MODULE_RESOLVE)` -- `[EXPERIMENTAL]` Inline functions -- `[EXPERIMENTAL]` Inline `enum.IntFlag`s -- Convert pass `remove_pass.py` +- Unfold IIFEs (`unfold_iife.py`, `UnfoldIIFE`) - `config.unfold_iife_lambdas`. Inlines immediately-invoked no-arg lambda calls: `(lambda: x)()` -> `x`. +- Remove dummy assignments (`remove_dummy_assignments.py`, `RemoveDummyAssignments`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.remove_dummy_assignments`. Removes self-assignments like `x = x` (same binding on both sides). +- Remove literal statements (`remove_literal_statements.py`, `RemoveLiteralStatements`) - `config.remove_literal_statements` (default off). Drops `Expr` statements that are just a literal constant. Leaves the first statement of a module/class/function body alone if it's a string literal (a docstring position) - `RemoveDocstrings` decides what happens to those. +- Remove docstrings (`remove_docstrings.py`, `RemoveDocstrings`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.remove_docstrings: bool | RemoveDocstringOptions` (default off; `RemoveDocstringOptions.also_modules` to also strip module docstrings). Preserves a docstring if the class/function is decorated with `@terser_hints.preserve_docstring` (`terser_hints` package, a runtime no-op marker). +- Combine imports (`combine_imports.py`, `CombineImports`) - `config.combine_imports`. Merges consecutive `import`/`from x import` statements where possible (never merges `from x import *`, and leaves a lone un-combinable import untouched rather than rebuilding it). +- Cleanup local imports (`cleanup_local_imports.py`, `CleanupLocalImports`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.cleanup_local_imports`. Removes unused imports inside a function/class body always; removes unused module-level imports too if `config.respect_all` and the name isn't exported (via `__all__`/no leading underscore). +- Remove annotations (`remove_annotations.py`, `RemoveAnnotations`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.remove_annotations: bool | RemoveAnnotationOptions`. +- Remove `type` statements (`remove_type_statements.py`, `RemoveTypeStatements`) - `config.remove_type_statements`. Drops `type X = ...` (PEP 695) alias statements. +- Remove typing classes (`remove_typing_classes.py`, `RemoveTypingClasses`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.remove_typing_classes`. Strips a bare `Protocol` base class, unless the class is decorated with `@typing.runtime_checkable` (needed for `isinstance` to keep working). +- Convert typing constructors (`convert_typing_constructors.py`, `ConvertTypingConstructors`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.convert_typing_constructors`. For simple field-only (no methods) `NamedTuple`/`TypedDict` classes: rewrites a `NamedTuple` class into `X = collections.namedtuple('X', (...), defaults=(...))` (adding `import collections` if needed); rewrites a `TypedDict` class into plain `dict` - the class is dropped and every `X(...)` construction site becomes `dict(...)`, but only when every use is a pure-keyword call (bails out and leaves the class alone otherwise). +- Remove `Generic`s (`remove_generics.py`, `RemoveGenerics`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.remove_generics`. Strips a bare (non-parametrized) `Generic` base class; leaves `Generic[T]` alone since that form has real `__class_getitem__` behavior. +- Remove `@overload`s (`remove_overloads.py`, `RemoveOverloads`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.remove_overloads`, always on when `config.remove_typing_decorators` is set. Drops `@typing.overload`-decorated stub defs, keeping the final undecorated implementation. +- Remove typing decorators (`remove_typing_decorators.py`, `RemoveTypingDecorators`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.remove_typing_decorators`. Strips `@typing.override`/`@typing.final`. +- Remove explicit `return None` (`remove_explicit_return_none.py`, `RemoveExplicitReturnNone`) - `config.remove_explicit_return_none`. Converts `return None` to bare `return`, and drops a trailing bare `return` from a function body. +- Fold constants (`constant_folding.py`, `FoldConstants`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.fold_constants` (numeric/bool folding), `config.remove_debug` (part of the same class). + - Constant arithmetic/unary operations on number/bool literals, kept only if the folded form is strictly shorter and round-trips to the same value and type. + - Boolean-identity comparisons: `x == True`/`x is True` -> `x`, `x == False`/`x is False` -> `not x` (and the `!=`/`is not` inverses). + - `__debug__` -> `True`/`False` literal (based on `config.optimize`), `typing.TYPE_CHECKING` -> `False`. + - f-strings: `f"{x}"` -> `str(x)`/`repr(x)`/`ascii(x)` (per conversion flag), `f"a{x}b{y}"` -> `'a' + str(x) + 'b' + str(y)` - only when no `format_spec` is used anywhere in the string. + - Collection-constructor literals: `list()` -> `[]`, `dict()` -> `{}`, `tuple()` -> `()`, `set([1, 2])`/`set((1, 2))` -> `{1, 2}` (only for a single list/tuple-literal argument). + - Not implemented: numeric-literal reformatting (`0b1` -> `1`, etc.), `\uXXXX` string unescaping (this is the printer's job, not a transform's - it always picks the shortest valid string representation already), and `sys.version_info`/`sys.platform` folding (no target-version/platform config exists to fold against). +- Convert `typing_extensions` (`convert_typing_extensions.py`, `ConvertTypingExtensions`) - `config.convert_typing_extensions`. Rewrites `from typing_extensions import X` to `from typing import X` for a fixed set of symbols long-stable in `typing`, only when every name in the statement is on that list. +- Remove dead blocks (`remove_dead_blocks.py`, `RemoveDeadBlocks`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - runs right after `FoldConstants` in registration order (needs its literal-bool output). Collapses `if :` into just the taken branch. +- Convert early exits (`convert_early_exits.py`, `ConvertEarlyExits`) - `config.convert_early_exits`. Merges `if cond: return a` immediately followed by `return b` into `return a if cond else b`. +- Convert to inline (`convert_to_inline.py`, `ConvertToInline`) - `config.convert_to_inline`. + - `if cond: func(x)` -> `cond and func(x)` + - `if fizz: foo()` / `else: bar()` -> `foo() if fizz else bar()` +- Convert to lambda (`convert_to_lambda.py`, `ConvertToLambda`) - `config.convert_to_lambda`. `def foo(...): return expr` -> `foo = lambda ...: expr` (only for a single `return ` body - a bare trailing expression isn't converted, since a lambda's value differs from a statement's implicit `None`). +- Convert dynamic attribute access (`convert_dynamic_attribute_access.py`, `ConvertDynamicAttributeAccess`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - `config.convert_dynamic_attribute_access`. `getattr(obj, "name")` -> `obj.name`, a bare `setattr(obj, "name", value)` statement -> `obj.name = value`. Ignored when: `name` isn't a constant string, `name` isn't a valid/non-keyword identifier, or `getattr` is called with a default value (3-arg form). +- Remove unnecessary base/meta classes (`remove_object_base.py`, `RemoveObject`) - `config.remove_explicit_base`. Removes `object` from a class's bases. +- Remove empty exception brackets (`remove_exception_brackets.py`, `RemoveExceptionBrackets`) `(Flags.REQUIRES_MODULE_RESOLVE)` - `config.remove_empty_exc_brackets`. `raise ValueError()` -> `raise ValueError` for built-in exceptions. +- Convert pass (`remove_pass.py`, `RemovePass`) - `config.convert_pass`. Removes `pass` statements, or converts to `0` if a suite would otherwise be empty. + +### Excluded + +- `[EXPERIMENTAL]` Inline functions - out of scope, not implemented. +- `[EXPERIMENTAL]` Inline `enum.IntFlag`s - out of scope, not implemented. ### After mangling -- Convert positional arguments `remove_posargs.py` `(Flags.INFLUENCES_MANGLING)` -- Remove `__all__` `(Flags.INFLUENCES_MANGLING)` +- Convert positional arguments (`remove_posargs.py`, `RemovePosArgs`) `(Flags.INFLUENCES_MANGLING)` - `config.convert_posargs`. Converts positional-only arguments to normal arguments. +- Remove `__all__` (`remove_all.py`, `RemoveAll`) `(Flags.INFLUENCES_MANGLING)` - `config.remove_dunder_all`. Drops the top-level `__all__` assignment. diff --git a/src/terser/_pipeline/transforms/__init__.py b/src/terser/_pipeline/transforms/__init__.py index 03a4aa7..7e2fc8f 100644 --- a/src/terser/_pipeline/transforms/__init__.py +++ b/src/terser/_pipeline/transforms/__init__.py @@ -1,4 +1,4 @@ -from ._suite import TransformCache +from ._suite import TransformCache, apply_pass from .contracts import Contracts from .combine_imports import CombineImports from .constant_folding import FoldConstants @@ -69,4 +69,4 @@ RemoveAll, ] -__all__ = ("TransformCache", "__transforms__") +__all__ = ("TransformCache", "apply_pass", "__transforms__") diff --git a/src/terser/_pipeline/transforms/__init__.pyi b/src/terser/_pipeline/transforms/__init__.pyi index 5c8a407..88579a2 100644 --- a/src/terser/_pipeline/transforms/__init__.pyi +++ b/src/terser/_pipeline/transforms/__init__.pyi @@ -1,6 +1,6 @@ -from ._suite import TransformCache, SuiteTransformer +from ._suite import TransformCache, SuiteTransformer, apply_pass from collections.abc import Iterable __transforms__: Iterable[type[SuiteTransformer]] -__all__ = ("TransformCache", "__transforms__",) +__all__ = ("TransformCache", "apply_pass", "__transforms__",) diff --git a/src/terser/_pipeline/transforms/_suite.py b/src/terser/_pipeline/transforms/_suite.py index 82c9954..5b284a3 100644 --- a/src/terser/_pipeline/transforms/_suite.py +++ b/src/terser/_pipeline/transforms/_suite.py @@ -11,6 +11,7 @@ from ..resolver.util import scope_ref_global if TYPE_CHECKING: + from collections.abc import Iterable from typing import Final, Self from terser.ast.ref import ContainsScope @@ -35,6 +36,26 @@ class TransformCache: passes: dict[type[SuiteTransformer], bool] = field(default_factory=dict) +def apply_pass(cache: TransformCache, module: ast.Module, transform_types: Iterable[type[SuiteTransformer]], flags_max: TransformerFlag | int) -> ast.Module: + """ + Apply every enabled transform up to `flags_max` once, recording per-transform + whether it changed the module into `cache.passes`. + + Callers should loop this until `not any(cache.passes.values())` (nothing + changed this pass) or `config.passes` is reached - `SuiteTransformer.__new__` + uses `cache.passes` to skip re-running a transform when nothing earlier in + `cache.transforms` changed since the last pass. + """ + cache.transforms = [t for t in transform_types if t.is_enabled(cache.config) and t.FLAGS <= flags_max] + + for transform in cache.transforms: + before = ast.dump(module) + module = transform(cache)(module) + cache.passes[transform] = ast.dump(module) != before + + return module + + class SuiteTransformer(NodeVisitor, ABC): """ Transform suites of instructions diff --git a/src/terser/project.py b/src/terser/project.py index 1b438d5..59d8836 100644 --- a/src/terser/project.py +++ b/src/terser/project.py @@ -111,22 +111,26 @@ async def __call__(self, /): for _, module in self.reporter.iter(collected, "Linking"): linker.link(module, project) - cache = transforms.TransformCache(self.config) + # Each module gets its own TransformCache - cache.passes tracks per-transform + # "did this change the module" for SuiteTransformer.__new__'s skip-unchanged + # optimization, which is meaningless if shared across independent module trees. + caches = {module: transforms.TransformCache(self.config) for module in collected} + for _ in self.reporter.range(self.config.passes, "Applying transforms"): + changed = False for module in collected: - for transform in transforms.__transforms__: - if not transform.is_enabled(self.config) or transform.FLAGS > 2: - continue - - module: ast.Module = transform(cache)(module) + cache = caches[module] + transforms.apply_pass(cache, module, transforms.__transforms__, 2) # mutates module in place + changed = changed or any(cache.passes.values()) - if not any(cache.passes.values()): + if not changed: break with self.reporter("Mangling"): mangler.mangle_globals(project, self.rename_globals, self.preserve_globals) for _, module in self.reporter.iter(collected, "Applying transforms"): + cache = caches[module] for transform in transforms.__transforms__: if not transform.is_enabled(self.config) or transform.FLAGS > 4: continue diff --git a/src/terser/terser.py b/src/terser/terser.py index a8194e1..a05828c 100644 --- a/src/terser/terser.py +++ b/src/terser/terser.py @@ -2,7 +2,7 @@ from ._minify import minify as __minify, unparse as __unparse from ._pipeline import transforms -from .ast import DummySpec, ast +from .ast import DummySpec from .config import TransformConfig from .project import ProjectMinifier @@ -46,11 +46,7 @@ def minify( cache = transforms.TransformCache(config) for _ in range(config.passes): - for transform in transforms.__transforms__: - if not transform.is_enabled(config) or transform.FLAGS > 4: - continue - - module: ast.Module = transform(cache)(module) + module = transforms.apply_pass(cache, module, transforms.__transforms__, 4) if not any(cache.passes.values()): break From b309c9abf3edc24266cbc7e016da044e4b181d1c Mon Sep 17 00:00:00 2001 From: Alpha Date: Fri, 24 Jul 2026 06:44:03 +0000 Subject: [PATCH 03/14] fix(transforms): drop f-string folding from FoldConstants f"{x}" -> str(x) assumed x isn't already a str, with no way to verify that statically - wrapping an already-str value just grows the source instead of shrinking it. Remove the whole visit_JoinedStr path rather than keep an optimization that can regress size. --- src/terser/_pipeline/transforms/README.md | 3 +- .../_pipeline/transforms/constant_folding.py | 33 ------------------- 2 files changed, 1 insertion(+), 35 deletions(-) diff --git a/src/terser/_pipeline/transforms/README.md b/src/terser/_pipeline/transforms/README.md index a0d1e87..d13ff24 100644 --- a/src/terser/_pipeline/transforms/README.md +++ b/src/terser/_pipeline/transforms/README.md @@ -22,9 +22,8 @@ All entries below are implemented and registered in `__transforms__` (`__init__. - Constant arithmetic/unary operations on number/bool literals, kept only if the folded form is strictly shorter and round-trips to the same value and type. - Boolean-identity comparisons: `x == True`/`x is True` -> `x`, `x == False`/`x is False` -> `not x` (and the `!=`/`is not` inverses). - `__debug__` -> `True`/`False` literal (based on `config.optimize`), `typing.TYPE_CHECKING` -> `False`. - - f-strings: `f"{x}"` -> `str(x)`/`repr(x)`/`ascii(x)` (per conversion flag), `f"a{x}b{y}"` -> `'a' + str(x) + 'b' + str(y)` - only when no `format_spec` is used anywhere in the string. - Collection-constructor literals: `list()` -> `[]`, `dict()` -> `{}`, `tuple()` -> `()`, `set([1, 2])`/`set((1, 2))` -> `{1, 2}` (only for a single list/tuple-literal argument). - - Not implemented: numeric-literal reformatting (`0b1` -> `1`, etc.), `\uXXXX` string unescaping (this is the printer's job, not a transform's - it always picks the shortest valid string representation already), and `sys.version_info`/`sys.platform` folding (no target-version/platform config exists to fold against). + - Not implemented: numeric-literal reformatting (`0b1` -> `1`, etc.), `\uXXXX` string unescaping (this is the printer's job, not a transform's - it always picks the shortest valid string representation already), `sys.version_info`/`sys.platform` folding (no target-version/platform config exists to fold against), and f-string folding (`f"{x}"` -> `str(x)` etc. - dropped: whether `x` is already a `str` can't be verified statically, so wrapping unconditionally risks growing the source instead of shrinking it). - Convert `typing_extensions` (`convert_typing_extensions.py`, `ConvertTypingExtensions`) - `config.convert_typing_extensions`. Rewrites `from typing_extensions import X` to `from typing import X` for a fixed set of symbols long-stable in `typing`, only when every name in the statement is on that list. - Remove dead blocks (`remove_dead_blocks.py`, `RemoveDeadBlocks`) `(Flags.REQUIRES_IMPORT_RESOLVE)` - runs right after `FoldConstants` in registration order (needs its literal-bool output). Collapses `if :` into just the taken branch. - Convert early exits (`convert_early_exits.py`, `ConvertEarlyExits`) - `config.convert_early_exits`. Merges `if cond: return a` immediately followed by `return b` into `return a if cond else b`. diff --git a/src/terser/_pipeline/transforms/constant_folding.py b/src/terser/_pipeline/transforms/constant_folding.py index 6d142e9..2197236 100644 --- a/src/terser/_pipeline/transforms/constant_folding.py +++ b/src/terser/_pipeline/transforms/constant_folding.py @@ -20,9 +20,6 @@ def _is_unshadowed_builtin(node, name: str) -> bool: return isinstance(binding, BuiltinBinding) and binding.name == name and not binding.is_redefined() -_CONVERSION_FUNCS = {-1: 'str', 115: 'str', 114: 'repr', 97: 'ascii'} - - def is_foldable_constant(node): """ Check if a node is a constant expression that can participate in folding. @@ -186,36 +183,6 @@ def visit_Attribute(self, node): node_ref = ref(node) return self.add_child(new_node, node_ref.parent, node_ref.namespace) - def visit_JoinedStr(self, node): - node.values = [self.visit(v) for v in node.values] - - if any(isinstance(v, ast.FormattedValue) and v.format_spec is not None for v in node.values): - return node - - terms = [] - for v in node.values: - if isinstance(v, ast.Constant): - terms.append(v) - continue - - func_name = _CONVERSION_FUNCS.get(v.conversion, None) - if func_name is None: - return node - - terms.append(ast.Call(func=ast.Name(id=func_name, ctx=ast.Load()), args=[v.value], keywords=[])) - - if not terms: - new_node = ast.Constant(value='') - elif len(terms) == 1 and isinstance(terms[0], ast.Call): - new_node = terms[0] - else: - new_node = terms[0] - for term in terms[1:]: - new_node = ast.BinOp(left=new_node, op=ast.Add(), right=term) - - node_ref = ref(node) - return self.add_child(new_node, node_ref.parent, node_ref.namespace) - def visit_Call(self, node): node.func = self.visit(node.func) node.args = [self.visit(a) for a in node.args] From 313f223af01d277fa2fd9890d78f8b4c35aac8e9 Mon Sep 17 00:00:00 2001 From: Alpha Date: Fri, 24 Jul 2026 07:00:30 +0000 Subject: [PATCH 04/14] fix(transforms): fix crash in RemoveLiteralStatements's __doc__ guard visit_Module accessed node.bindings directly (not a real attribute - NodeRef metadata is only reachable via ref(node)) and compared binding.spec, a field Binding doesn't have. Both were always broken; this only surfaced now because remove_literal_statements defaults off and nobody had exercised it with the option enabled yet. Also: this transform is FLAGS=0 (runs before resolver.resolve()/bind()), so binding data wouldn't have existed yet even with the attribute access fixed - ref(node).bindings would just be empty. Replaced the check with a plain structural scan for a module-level `__doc__ = ...` assignment, which is all that's actually available at this stage and matches what the check was trying to detect. --- .../transforms/remove_literal_statements.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/terser/_pipeline/transforms/remove_literal_statements.py b/src/terser/_pipeline/transforms/remove_literal_statements.py index d66be05..2e8a3a2 100644 --- a/src/terser/_pipeline/transforms/remove_literal_statements.py +++ b/src/terser/_pipeline/transforms/remove_literal_statements.py @@ -19,6 +19,20 @@ def _doc_in_module(module): return True +def _defines_dunder_doc(module): + # FLAGS = 0, this runs before resolver.resolve()/bind() - no binding info + # exists yet, so this has to be a plain structural scan for a module-level + # `__doc__ = ...` / `__doc__: ... = ...` assignment + for stmt in module.body: + if isinstance(stmt, ast.Assign) and any(isinstance(t, ast.Name) and t.id == '__doc__' for t in stmt.targets): + return True + + if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name) and stmt.target.id == '__doc__': + return True + + return False + + class RemoveLiteralStatements(SuiteTransformer): """ Remove literal expressions from the code @@ -37,10 +51,9 @@ def __call__(self, node): return self.visit(node) def visit_Module(self, node): - for binding in node.bindings: - if binding.spec == '__doc__': - node.body = [self.visit(a) for a in node.body] - return node + if _defines_dunder_doc(node): + node.body = [self.visit(a) for a in node.body] + return node node.body = self.suite(node.body, parent=node) return node From b8610069093d5db757dbce105dcf475c458681f1 Mon Sep 17 00:00:00 2001 From: Alpha Date: Fri, 24 Jul 2026 07:11:33 +0000 Subject: [PATCH 05/14] fix(mangler): register nodes created during literal hoisting HoistedBinding.rename() (mangler/_constants.py) builds three kinds of fresh nodes when hoisting a repeated literal to a local variable: a Load-context Name at each original occurrence (via replace()), a Store-context Name for the new assignment's target, and the Assign statement itself. None of them ever got a real NodeRef/binding: replace() attaches a NodeRef but never calls add_reference, so ref(node).binding raised AttributeError on the wrapper; the Assign and its target were never passed through NodeRef.new/add_child at all, so ref() on them raised AttributeError on the raw node itself (no wrapper attribute present). Both were always broken, but every existing transform happened to never call ref(...).binding on these particular nodes. RemoveDummyAssignments (added this session) is the first to unconditionally check the binding of both sides of every Assign, which is what surfaced the crash - reproduced via `terser --output ...` on a source with a literal repeated often enough to trigger hoisting. Also apply the same "namespace may be orphaned by a deleted subtree" defensive check already added to reserve_name() to its sibling is_available() in mangler/_locals.py - same crash class, just not yet hit by an actual repro. --- src/terser/_pipeline/mangler/_constants.py | 31 +++++++++++++++------- src/terser/_pipeline/mangler/_locals.py | 11 +++++++- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/terser/_pipeline/mangler/_constants.py b/src/terser/_pipeline/mangler/_constants.py index 6f37647..bee5226 100644 --- a/src/terser/_pipeline/mangler/_constants.py +++ b/src/terser/_pipeline/mangler/_constants.py @@ -64,15 +64,28 @@ def additional_byte_cost(self): def rename(self, new_name): - for node in self.references: - replace(node, ast.Name(id=new_name, ctx=ast.Load())) - - self._local_namespace.body = list( - insert( - self._local_namespace.body, - ast.Assign(targets=[ast.Name(id=new_name, ctx=ast.Store())], value=self._value_node), - ) - ) + # snapshot first - add_reference (below) appends to this same list, and we + # only want to replace the references that existed going into this rename + old_references = list(self.references) + + for node in old_references: + new_node = ast.Name(id=new_name, ctx=ast.Load()) + replace(node, new_node) + self.add_reference(new_node) + + # self._value_node was one of the just-replaced references (the first occurrence + # found) - it's an existing, already-registered node, just moving to a new parent + target = ast.Name(id=new_name, ctx=ast.Store()) + new_stmt = ast.Assign(targets=[target], value=self._value_node) + + NodeRef.new(new_stmt, self._local_namespace) + ref(new_stmt).namespace = self._local_namespace + NodeRef.new(target, new_stmt) + ref(target).namespace = self._local_namespace + ref(self._value_node).parent = new_stmt + self.add_reference(target) + + self._local_namespace.body = list(insert(self._local_namespace.body, new_stmt)) self._name = new_name diff --git a/src/terser/_pipeline/mangler/_locals.py b/src/terser/_pipeline/mangler/_locals.py index 4718e93..40131c7 100644 --- a/src/terser/_pipeline/mangler/_locals.py +++ b/src/terser/_pipeline/mangler/_locals.py @@ -206,7 +206,16 @@ def is_available(self, name, reservation_scope): """ - return all(name not in ref(namespace).assigned_names for namespace in reservation_scope) + def unreserved(namespace): + namespace_ref = ref(namespace) + if not hasattr(namespace_ref, 'assigned_names'): + # namespace is no longer reachable from the module root (a transform + # deleted the subtree it belonged to) - nothing reserves anything there + return True + + return name not in namespace_ref.assigned_names + + return all(unreserved(namespace) for namespace in reservation_scope) def assign(self, namespace, binding, *, prefix=''): """ From 243c0a567227bc64d11ee1aeb08cdabd326a633a Mon Sep 17 00:00:00 2001 From: Alpha Date: Fri, 24 Jul 2026 07:30:42 +0000 Subject: [PATCH 06/14] fix(mangler): register the keyword-param alias assignment; harden ref() call sites Third instance of the same class of bug: NameBinding.rename() (resolver/binding.py) inserts an alias assignment (`A = original_param_name`) at the top of a function body whenever a keyword-callable parameter gets shortened - the original name has to stay (call sites may use it as a keyword), so the short name is bound separately. The new Assign and its Name nodes were never passed through NodeRef.new, so ref() on them raised AttributeError on the raw node, exactly like the two mangler bugs fixed already this session. Reproduced via `terser --output ...` on a source with a method taking several positional-or-keyword parameters. Registered the new nodes the same way as the earlier _constants.py fix - the target gets no binding (it's a distinct identity from the parameter, so aliasing it to the same binding would make RemoveDummyAssignments mistake the alias for a no-op `x = x` and delete it, leaving the shortened name undefined everywhere it's used); the value does get one, since it's a genuine read of the parameter. Given this is now the third occurrence of "a mangler-synthesized node was never fully registered," also hardened every place that unconditionally calls ref(node).binding on an arbitrary Name it encounters during traversal (RemoveDummyAssignments, FoldConstants/ConvertDynamicAttributeAccess's _is_unshadowed_builtin, qualified_name) to treat an unregistered node as unresolvable instead of crashing - a general safety net for whichever mangler quirk turns up next, rather than chasing each one individually. --- src/terser/_pipeline/resolver/binding.py | 26 ++++++++++++------- .../_pipeline/transforms/constant_folding.py | 7 ++++- .../convert_dynamic_attribute_access.py | 7 ++++- .../transforms/remove_dummy_assignments.py | 13 +++++++++- src/terser/utils/imports.py | 9 ++++++- 5 files changed, 49 insertions(+), 13 deletions(-) diff --git a/src/terser/_pipeline/resolver/binding.py b/src/terser/_pipeline/resolver/binding.py index b01b3c3..f042bd8 100644 --- a/src/terser/_pipeline/resolver/binding.py +++ b/src/terser/_pipeline/resolver/binding.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, override from terser.ast import ast, ref +from terser.ast.ref._node import NodeRef from .util import arg_rename_in_place, insert if TYPE_CHECKING: @@ -386,15 +387,22 @@ def rename(self, new_name: str): node.name = new_name if func_namespace_binding: - func_namespace_binding.body = list( - insert( - func_namespace_binding.body, - ast.Assign( - targets=[ast.Name(id=new_name, ctx=ast.Store())], - value=ast.Name(id=self._name, ctx=ast.Load()), - ), - ) - ) + # a keyword-callable parameter can't be renamed in place (would break call + # sites), so it keeps its original name and gets aliased to the new short + # name via an assignment at the top of the function body instead + target = ast.Name(id=new_name, ctx=ast.Store()) + value = ast.Name(id=self._name, ctx=ast.Load()) + new_stmt = ast.Assign(targets=[target], value=value) + + NodeRef.new(new_stmt, func_namespace_binding) + ref(new_stmt).namespace = func_namespace_binding + NodeRef.new(target, new_stmt) + ref(target).namespace = func_namespace_binding + NodeRef.new(value, new_stmt) + ref(value).namespace = func_namespace_binding + self.add_reference(value) # reads the (still current) self._name - another reference to this binding + + func_namespace_binding.body = list(insert(func_namespace_binding.body, new_stmt)) self._name = new_name diff --git a/src/terser/_pipeline/transforms/constant_folding.py b/src/terser/_pipeline/transforms/constant_folding.py index 2197236..c38c249 100644 --- a/src/terser/_pipeline/transforms/constant_folding.py +++ b/src/terser/_pipeline/transforms/constant_folding.py @@ -16,7 +16,12 @@ def _is_unshadowed_builtin(node, name: str) -> bool: if not isinstance(node, ast.Name): return False - binding = ref(node).binding + try: + binding = ref(node).binding + except AttributeError: + # some mangler-synthesized nodes are never fully registered with a binding + return False + return isinstance(binding, BuiltinBinding) and binding.name == name and not binding.is_redefined() diff --git a/src/terser/_pipeline/transforms/convert_dynamic_attribute_access.py b/src/terser/_pipeline/transforms/convert_dynamic_attribute_access.py index 051bb66..fd42290 100644 --- a/src/terser/_pipeline/transforms/convert_dynamic_attribute_access.py +++ b/src/terser/_pipeline/transforms/convert_dynamic_attribute_access.py @@ -19,7 +19,12 @@ def _is_unshadowed_builtin(node: ast.expr, name: str) -> bool: if not isinstance(node, ast.Name): return False - binding = ref(node).binding + try: + binding = ref(node).binding + except AttributeError: + # some mangler-synthesized nodes are never fully registered with a binding + return False + return isinstance(binding, BuiltinBinding) and binding.name == name and not binding.is_redefined() diff --git a/src/terser/_pipeline/transforms/remove_dummy_assignments.py b/src/terser/_pipeline/transforms/remove_dummy_assignments.py index 1b49c90..6a1bb7f 100644 --- a/src/terser/_pipeline/transforms/remove_dummy_assignments.py +++ b/src/terser/_pipeline/transforms/remove_dummy_assignments.py @@ -5,6 +5,13 @@ from ._suite import SuiteTransformer, TransformerFlag +def _binding_of(node): + try: + return ref(node).binding + except AttributeError: + return None + + class RemoveDummyAssignments(SuiteTransformer): """ Remove self-assignments like `x = x` @@ -33,4 +40,8 @@ def _is_dummy(self, node) -> bool: if not isinstance(target, ast.Name) or not isinstance(value, ast.Name): return False - return ref(target).binding is ref(value).binding + target_binding = _binding_of(target) + # some mangler-synthesized nodes are never fully registered with a NodeRef/binding + # (e.g. an aliasing assignment for a keyword-callable renamed parameter) - if we + # can't resolve both sides, we can't prove this is a genuine dummy assignment + return target_binding is not None and target_binding is _binding_of(value) diff --git a/src/terser/utils/imports.py b/src/terser/utils/imports.py index 5160cd2..aa024ea 100644 --- a/src/terser/utils/imports.py +++ b/src/terser/utils/imports.py @@ -20,7 +20,14 @@ def qualified_name(node: ast.expr, /) -> str | None: if not isinstance(node, ast.Name): return None - binding = ref(node).binding + # some mangler-synthesized nodes are never fully registered with a NodeRef/binding + # (e.g. an aliasing assignment for a keyword-callable renamed parameter) - treat + # those as unresolvable rather than crashing + try: + binding = ref(node).binding + except AttributeError: + return None + if not isinstance(binding, ImportBinding) or not binding.name: return None From e2fd97458c7226241a81509be24d851b26c46988 Mon Sep 17 00:00:00 2001 From: Alpha Date: Fri, 24 Jul 2026 08:11:23 +0000 Subject: [PATCH 07/14] fix(preprocessor): stop corrupting multi-line strings that contain '#' lines preprocess() scans source line-by-line at raw text level with no notion of string literals: any line whose stripped text starts with '#' is treated as a directive/comment line, and if it doesn't match an if/elif/else/endif directive, it's silently dropped from the output (never appended). This is fine for real comments, but a multi-line string whose content happens to contain lines starting with '#' - e.g. a string constant holding an example/generated Python code snippet, itself full of real comments - gets those lines deleted too, corrupting the string and frequently producing an "unterminated triple-quoted string literal" SyntaxError downstream (repro: beartype's _data/code/pep/datacodepep525.py, a ~250 line file with a large f-string constant full of comment-shaped lines; py_compile handles it fine, terser's preprocessor mangled it down to 90 lines). Added _multiline_string_body_lines(), which tokenizes the source once and returns every line number that falls inside a multi-line string/f-string body (excluding the opening line, which may have real code before the string starts). preprocess() now passes those lines through untouched regardless of what they look like, leaving the existing directive/comment handling for genuine code as-is. --- src/terser/_pipeline/preprocessor.py | 46 +++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/terser/_pipeline/preprocessor.py b/src/terser/_pipeline/preprocessor.py index bb2a9f8..5b76c1d 100644 --- a/src/terser/_pipeline/preprocessor.py +++ b/src/terser/_pipeline/preprocessor.py @@ -1,4 +1,6 @@ +import io import re +import tokenize from typing import TYPE_CHECKING @@ -7,6 +9,39 @@ from typing import Final +_STRING_TOKEN_TYPES: Final[frozenset[int]] = frozenset( + {tokenize.STRING} + | ({tokenize.FSTRING_START, tokenize.FSTRING_MIDDLE, tokenize.FSTRING_END} if hasattr(tokenize, "FSTRING_START") else set()) +) + + +def _multiline_string_body_lines(source: str) -> frozenset[int]: + """ + 1-indexed source line numbers that fall inside the body of a multi-line + string/f-string literal (its opening line is excluded, since that line may + have real code before the string starts). + + Directive/comment handling in `preprocess` works line-by-line on raw text + with no awareness of string literals - without this, a multi-line string + whose content happens to contain lines starting with `#` (e.g. a string + constant full of example Python source, itself full of real comments) gets + its "comment" lines silently dropped, corrupting the string. + """ + body_lines: set[int] = set() + + try: + tokens = tokenize.generate_tokens(io.StringIO(source).readline) + for tok in tokens: + if tok.type in _STRING_TOKEN_TYPES and tok.start[0] != tok.end[0]: + body_lines.update(range(tok.start[0] + 1, tok.end[0] + 1)) + except (tokenize.TokenError, SyntaxError): + # if the source doesn't even tokenize, let the later real parse step + # raise a proper error - here, just fall back to the old unaware behavior + return frozenset() + + return frozenset(body_lines) + + __DIRECTIVES: Final[Mapping[str, Mapping[bool, re.Pattern]]] = { "if": { True: re.compile(r"^#\s?if ([A-Za-z_][A-Za-z0-9_]*)$"), @@ -39,12 +74,21 @@ def preprocess(source: str, defines: Mapping[str, bool] | None, strict: bool = F shebang = lines.pop(0) if lines[0].startswith("#!") else None defines: Mapping[str, bool] = defines or {} + string_body_lines = _multiline_string_body_lines(source) + line_offset = 2 if shebang is not None else 1 + # Directive evaluation output = [] stack: list[tuple[bool, bool | None]] = [] keeping = lambda: all(s[0] for s in stack) - for line in lines: + for i, line in enumerate(lines): + if (i + line_offset) in string_body_lines: + # inside the body of a multi-line string/f-string - pass through + # untouched, whatever it looks like isn't a real comment/directive + output.append(line) + continue + stripped = line.strip() if not stripped.startswith('#'): From 66455538dc11e9e993e9414e08e544ca32a21005 Mon Sep 17 00:00:00 2001 From: Alpha Date: Sat, 25 Jul 2026 15:48:07 +0000 Subject: [PATCH 08/14] wip: some changes for debugging --- src/terser/_minify.py | 5 ++++- src/terser/_pipeline/mangler/_locals.py | 8 ++++---- src/terser/_pipeline/resolver/binder/__init__.py | 5 +++-- src/terser/cli/_tqdm.py | 14 ++++++++++++++ src/terser/cli/main.py | 4 ++-- src/terser/project.py | 4 ++-- src/terser/terser.py | 4 ++-- 7 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/terser/_minify.py b/src/terser/_minify.py index e493fc2..049c367 100644 --- a/src/terser/_minify.py +++ b/src/terser/_minify.py @@ -53,12 +53,15 @@ def minify( /, config: TransformConfig, *, + link_imports: bool = False, strict: bool = False, defines: dict[str, bool] | None = None, rename: bool = True, preserved_names: list[str] | None = None, hoist_literals: bool = True, ) -> tuple[ast.Module, str | None]: + getattr(task, "_test", lambda _: None)(str(spec)) + with task("Preprocessing sources"): source, shebang = preprocessor.preprocess(source, defines, strict) @@ -73,7 +76,7 @@ def minify( with task("Resolving names"): resolver.resolve(module) - resolver.bind(module) + resolver.bind(module, link_imports=link_imports) cache = transforms.TransformCache(config) for _ in task("Applying transforms", range(config.passes)): diff --git a/src/terser/_pipeline/mangler/_locals.py b/src/terser/_pipeline/mangler/_locals.py index 40131c7..c30927b 100644 --- a/src/terser/_pipeline/mangler/_locals.py +++ b/src/terser/_pipeline/mangler/_locals.py @@ -264,7 +264,7 @@ def __call__(self, module, prefix_globals=False, reserved_globals=None): return module -def mangle_locals(module, rename_locals=True, preserve_locals=None): +def mangle_locals(module: ast.Module, rename_locals: bool = True, preserved_names: list[str] | None = None): """ Mangle locals/nonlocals - names bound in function and class namespaces @@ -275,11 +275,11 @@ def mangle_locals(module, rename_locals=True, preserve_locals=None): :param module: The module to mangle locals in :type module: :class:`ast.Module` :param bool rename_locals: If local names may be renamed - :param preserve_locals: Local names to leave unchanged - :type preserve_locals: list[str] | None + :param preserved_names: Local names to leave unchanged + :type preserved_names: list[str] | None """ - allow_rename_locals(module, rename_locals, preserve_locals) + allow_rename_locals(module, rename_locals, preserved_names) add_assigned(module) diff --git a/src/terser/_pipeline/resolver/binder/__init__.py b/src/terser/_pipeline/resolver/binder/__init__.py index 6bb19a9..ccbba47 100644 --- a/src/terser/_pipeline/resolver/binder/__init__.py +++ b/src/terser/_pipeline/resolver/binder/__init__.py @@ -9,10 +9,11 @@ from ast import Module -def bind(module: Module): +def bind(module: Module, /, link_imports: bool = False): __resolve__all__(module) __mark_exports(module) - __resolve_imports(module) + if link_imports: + __resolve_imports(module) __bind(module) diff --git a/src/terser/cli/_tqdm.py b/src/terser/cli/_tqdm.py index 944ca3e..816e15b 100644 --- a/src/terser/cli/_tqdm.py +++ b/src/terser/cli/_tqdm.py @@ -98,7 +98,14 @@ def _step_context(self, message: str, /) -> StepContext: phase, self.__cur = self.__ctx._tg.steps[self.__cur], self.__cur + 1 return _TqdmStepContext(_Context(self.__pv, phase), message) + def _test(self, path: str): + self.__path = path + with self.__pv.active_lock: + self.__pv.active.append(self.__path) + def done(self, /) -> None: + with self.__pv.active_lock: + self.__pv.active.remove(self.__path) self.__ctx.close() @@ -113,6 +120,10 @@ def update(self, n: int, /): self.__parent.display(pos=0) self.__bar.display(pos=1) + active_str = f"Active ({len(self.active)}): " + + self.__bar.display(active_str + ', '.join(self.active)[:120 - len(active_str)], pos=2) + def end_status(self, /): pass @@ -127,6 +138,9 @@ def __init__(self, context: _Context[TqdmDebugTaskGraph.Task], parent: tqdm, mes self.__parent = parent self.__lock = Lock() + self.active_lock = Lock() + self.active = [] + def __enter__(self) -> None: self.__bar: tqdm = tqdm(total=len(self.__ctx._tg) * self.__ctx._tg.steps_size, leave=False) self.__ctx.enter(self.__msg) diff --git a/src/terser/cli/main.py b/src/terser/cli/main.py index 5c3648a..d43ba9c 100644 --- a/src/terser/cli/main.py +++ b/src/terser/cli/main.py @@ -60,8 +60,8 @@ def main(): preserve_shebang=args.preserve_shebang, prefer_single_line=args.prefer_single_line, hoist_literals=args.mangling_options.hoist_literals, - rename_locals=args.mangling_options.rename_locals, - preserve_locals=local, + rename=args.mangling_options.rename_locals, + preserved_names=local, ) except UnbeneficialMinificationError: # Use original source when minification isn't beneficial diff --git a/src/terser/project.py b/src/terser/project.py index e1cd54e..bb66192 100644 --- a/src/terser/project.py +++ b/src/terser/project.py @@ -163,8 +163,8 @@ async def __minify_modules(self, /) -> tuple[list[ast.Module], dict[str, ModuleR def __run(task: Task, source: str, spec: ModuleSpec, /): local = sorted(preserved_names(str(spec), self.preserve_locals)) return minify( - task, source, spec, - self.__config, + task, source, spec, self.__config, + link_imports=True, hoist_literals=self.hoist_literals, rename=self.rename_locals, preserved_names=local, diff --git a/src/terser/terser.py b/src/terser/terser.py index a05828c..5bbd745 100644 --- a/src/terser/terser.py +++ b/src/terser/terser.py @@ -1,4 +1,4 @@ -from alpha93.progression.tasks import Task +from alpha93.progression.headless import _EmptyTaskProvider as _TaskProvider from ._minify import minify as __minify, unparse as __unparse from ._pipeline import transforms @@ -42,7 +42,7 @@ def minify( :rtype: str """ - module, shebang = __minify(Task(), source, DummySpec(path), config, **kwargs) + module, shebang = __minify(_TaskProvider.EmptyTask(), source, DummySpec(path), config, **kwargs) cache = transforms.TransformCache(config) for _ in range(config.passes): From 57669cbaeed7a3b9f9810dbeda2e5b414635c383 Mon Sep 17 00:00:00 2001 From: Alpha Date: Sat, 25 Jul 2026 16:48:27 +0000 Subject: [PATCH 09/14] fix: `__init__.py` not written properly --- src/terser/_pipeline/transforms/_suite.py | 2 +- src/terser/ast/ref/_module/_spec.py | 2 +- src/terser/project.py | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/terser/_pipeline/transforms/_suite.py b/src/terser/_pipeline/transforms/_suite.py index 5b284a3..79b79af 100644 --- a/src/terser/_pipeline/transforms/_suite.py +++ b/src/terser/_pipeline/transforms/_suite.py @@ -246,7 +246,7 @@ def nearest_function_namespace(node: ast.AST) -> ContainsScope: :param node: The node to get the function namespace of """ - if isinstance(node, (ast.FunctionDef, ast.Module, ast.AsyncFunctionDef)): + if isinstance(node, (ast.FunctionDef, ast.Module, ast.AsyncFunctionDef, ast.Lambda)): return node return nearest_function_namespace(ref(node).parent) diff --git a/src/terser/ast/ref/_module/_spec.py b/src/terser/ast/ref/_module/_spec.py index d55b529..89f16eb 100644 --- a/src/terser/ast/ref/_module/_spec.py +++ b/src/terser/ast/ref/_module/_spec.py @@ -76,7 +76,7 @@ class PackageSpec(ModuleSpec): def __init__(self, unresolved: ModuleSpec, parent: PackageSpec | None = None): assert str(unresolved).endswith(".__init__") - super().__init__(str(unresolved).rstrip(".__init__")) + super().__init__(str(unresolved).removesuffix(".__init__")) self.__path = unresolved.path.parent self.__parent = parent self.__children = {} diff --git a/src/terser/project.py b/src/terser/project.py index bb66192..ac1f4f4 100644 --- a/src/terser/project.py +++ b/src/terser/project.py @@ -11,6 +11,7 @@ from ._pipeline import PathProvider, Pipeline, linker, mangler, transforms from ._pipeline.mangler.util import preserved_names from .ast import ref +from .ast.ref._module import PackageSpec if TYPE_CHECKING: import ast @@ -200,6 +201,11 @@ async def module(node: ast.Module, /): if self.__output is None: dest = spec.path + elif isinstance(spec, PackageSpec): + # PackageSpec's dotted name doesn't include the "__init__" component, + # so it needs its own path instead of the generic name -> path mapping below. + dest = self.__output / str(spec).replace('.', Path.parser.sep) / spec.path.name + await dest.parent.mkdir(parents=True, exist_ok=True) else: dest = self.__output / str(spec).replace('.', Path.parser.sep) dest = dest.with_suffix(spec.path.suffix) @@ -224,6 +230,7 @@ async def binary(path: Path, /): def wrap[T](func: Callable[[T], Awaitable[None]]) -> Callable[[T], Callable[[Task], Awaitable[None]]]: def wrapper(t: T) -> Callable[[Task], Awaitable[None]]: async def runner(task: Task, /): + getattr(task, "_test", lambda _: None)("") await func(t) task.done() return runner From e36beb9039cb59e264a611a0de2e535884ee31c6 Mon Sep 17 00:00:00 2001 From: Alpha Date: Sat, 25 Jul 2026 19:25:26 +0000 Subject: [PATCH 10/14] fix: --- .../_pipeline/transforms/constant_folding.py | 28 +++++++++++++++++++ .../_pipeline/transforms/remove_debug.py | 22 +++++++++++---- src/terser/config.py | 7 +++-- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/terser/_pipeline/transforms/constant_folding.py b/src/terser/_pipeline/transforms/constant_folding.py index c38c249..7eee9cb 100644 --- a/src/terser/_pipeline/transforms/constant_folding.py +++ b/src/terser/_pipeline/transforms/constant_folding.py @@ -25,6 +25,26 @@ def _is_unshadowed_builtin(node, name: str) -> bool: return isinstance(binding, BuiltinBinding) and binding.name == name and not binding.is_redefined() +def is_provably_bool(node) -> bool: + """ + Check if a node's value is guaranteed to be a bool, syntactically. + + Used to guard `X is True`/`X is False`-style folding: swapping an identity/equality + check for a bare truthiness check is only sound when `X` can't be some other + truthy/falsy-but-not-actually-bool value. + """ + if is_constant_node(node, ast.NameConstant) and isinstance(node.value, bool): + return True + + if isinstance(node, ast.Compare): + return True + + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return True + + return False + + def is_foldable_constant(node): """ Check if a node is a constant expression that can participate in folding. @@ -164,6 +184,14 @@ def visit_Compare(self, node): return node bool_value, other = (left.value, right) if left_bool else (right.value, left) + + # `X is True` -> `X` (and similar) is only sound if `X` is itself + # guaranteed to be a bool - otherwise it swaps an identity/equality + # check for a truthiness check, which differ for any non-bool value + # (e.g. `0 is False` is False, but `not 0` is True). + if not is_provably_bool(other): + return node + negate = bool_value == isinstance(node.ops[0], (ast.NotEq, ast.IsNot)) new_node = ast.UnaryOp(op=ast.Not(), operand=other) if negate else other diff --git a/src/terser/_pipeline/transforms/remove_debug.py b/src/terser/_pipeline/transforms/remove_debug.py index 7beafbb..8a941b5 100644 --- a/src/terser/_pipeline/transforms/remove_debug.py +++ b/src/terser/_pipeline/transforms/remove_debug.py @@ -1,4 +1,4 @@ -from terser.ast import ast, is_constant_node +from terser.ast import ast, is_constant_node, ref from ._suite import SuiteTransformer @@ -59,13 +59,23 @@ def is_truthy_debug_comparison(node: ast.If): return False def suite(self, node_list, parent): - - without_debug = [self.visit(a) for a in filter(lambda n: not self.__can_remove(n), node_list)] - - if len(without_debug) == 0: + result = [] + for node in node_list: + if self.__can_remove(node): + # An `else` branch is production code that must survive even when the + # `__debug__` branch itself is stripped. + if node.orelse: + for stmt in node.orelse: + ref(stmt).parent = parent + result.extend(self.suite(node.orelse, parent)) + continue + + result.append(self.visit(node)) + + if len(result) == 0: if isinstance(parent, ast.Module): return [] else: return [self.add_child(ast.Expr(value=ast.Num(0)), parent=parent)] - return without_debug + return result diff --git a/src/terser/config.py b/src/terser/config.py index 2bd49cb..d434721 100644 --- a/src/terser/config.py +++ b/src/terser/config.py @@ -69,8 +69,11 @@ class TransformConfig: unfold_iife_lambdas: bool = True """Inline immediately-invoked no-arg lambda calls, e.g. `(lambda: x)()` -> `x`""" - remove_type_statements: bool = True - """Remove `type X = ...` alias statements""" + remove_type_statements: bool = False + """Remove `type X = ...` alias statements. Unsafe by default: this runs pre-transform + (before even per-module name resolution), so it can't tell whether the alias is + actually imported/used by another module at runtime - only enable this if no `type` + statement in the project is relied on outside of typing contexts""" convert_early_exits: bool = True """Merge `if cond: return a` followed by `return b` into `return a if cond else b`""" From 9291a8b43a14e83a0cd751b537a19fcd00b6c739 Mon Sep 17 00:00:00 2001 From: Claude Sonnet 5 Date: Sun, 26 Jul 2026 16:13:49 +0000 Subject: [PATCH 11/14] fix: don't break code that's only visible from other modules Several per-module transforms (FLAGS <= REQUIRES_IMPORT_RESOLVE) run before project-wide linking, so they can't see how a name is used outside the module they're rewriting. Tighten them to stay correct in that blind spot: - convert_typing_constructors: don't drop a TypedDict class definition when it's exported (`__all__` or otherwise public) - it may only be referenced from other modules, which this pass can't observe. - remove_annotations: keep `Annotated[...]` annotations regardless of what they wrap, since the metadata (e.g. pydantic `Field(...)`) is often consumed at runtime and can't be reconstructed from a bare type or signature default. Plain string forward-ref annotations still get stripped unconditionally. - remove_dunder_all / remove_typing_classes: default to off. Both are only safe when no other module does `from x import *` or subclasses a stripped `Protocol`, which this pass has no way to check. - remove_all: keep the `__all__` assignment if anything else in the module still references the name (e.g. `__all__.append(...)`). - arg_rename_in_place: stop renaming `*args`/`**kwargs` parameter names in place - code may introspect them by name. - project.py: don't assert on a missing `--output` when writing a binary (FFI) file; skip it instead. --- src/terser/_pipeline/resolver/util.py | 13 +++++------ .../transforms/convert_typing_constructors.py | 7 +++++- src/terser/_pipeline/transforms/remove_all.py | 16 ++++++++++++- .../transforms/remove_annotations.py | 23 +++++++++++++++---- src/terser/config.py | 13 +++++++---- src/terser/project.py | 3 ++- 6 files changed, 56 insertions(+), 19 deletions(-) diff --git a/src/terser/_pipeline/resolver/util.py b/src/terser/_pipeline/resolver/util.py index 00b4150..f168709 100644 --- a/src/terser/_pipeline/resolver/util.py +++ b/src/terser/_pipeline/resolver/util.py @@ -39,9 +39,12 @@ def arg_rename_in_place(node: ast.AST, /) -> bool: """ Can this argument node by safely renamed - 'self', 'cls', 'args', and 'kwargs' are not commonly referenced by the caller, so - can be safely renamed. Comprehension arguments are not accessible from outside, so - can be renamed. + 'self' and 'cls' are not commonly referenced by the caller, so can be safely renamed. + Comprehension arguments are not accessible from outside, so can be renamed. + + '*args'/'**kwargs' names are deliberately NOT renamed in place: code may introspect + them by name (e.g. `inspect.signature(f).parameters['extra']`), which renaming would + silently break. If the argument is positional-only, it can be safely renamed @@ -70,10 +73,6 @@ def arg_rename_in_place(node: ast.AST, /) -> bool: # mangler 'cls' return True - if func.args.vararg is node or func.args.kwarg is node: - # starargs - return True - if hasattr(func.args, "posonlyargs") and node in func.args.posonlyargs: return True diff --git a/src/terser/_pipeline/transforms/convert_typing_constructors.py b/src/terser/_pipeline/transforms/convert_typing_constructors.py index 56d69dc..8179b3e 100644 --- a/src/terser/_pipeline/transforms/convert_typing_constructors.py +++ b/src/terser/_pipeline/transforms/convert_typing_constructors.py @@ -3,6 +3,7 @@ from terser.ast import ast, ref from terser.config import TransformConfig from terser.utils.imports import qualified_name +from ..resolver.util import insert from ._suite import SuiteTransformer, TransformerFlag _NAMEDTUPLE_NAMES = ("typing.NamedTuple", "typing_extensions.NamedTuple") @@ -53,7 +54,8 @@ def visit_Module(self, node: ast.Module): node.body = self.suite(node.body, parent=node) if self._needs_collections_import and not self._collections_imported: - node.body.insert(0, self.add_child(ast.Import(names=[ast.alias(name='collections', asname=None)]), parent=node)) + import_stmt = self.add_child(ast.Import(names=[ast.alias(name='collections', asname=None)]), parent=node) + node.body = list(insert(node.body, import_stmt)) return node @@ -91,6 +93,9 @@ def _convert_named_tuple(self, node: ast.ClassDef, fields: list[ast.AnnAssign]): def _convert_typed_dict(self, node: ast.ClassDef): binding = ref(node).binding + if binding.exported: + return _UNSAFE # may be used from other modules, which this per-module pass can't see + other_refs = [r for r in binding.references if r is not node] for r in other_refs: diff --git a/src/terser/_pipeline/transforms/remove_all.py b/src/terser/_pipeline/transforms/remove_all.py index f67cd64..99e1509 100644 --- a/src/terser/_pipeline/transforms/remove_all.py +++ b/src/terser/_pipeline/transforms/remove_all.py @@ -25,5 +25,19 @@ def is_enabled(cls, config: TransformConfig, /) -> bool: @override def visit_Module(self, node: ast.Module): - node.body = [stmt for stmt in node.body if not _is_dunder_all_assign(stmt)] + assigns = [stmt for stmt in node.body if _is_dunder_all_assign(stmt)] + if not assigns: + return node + + # Removing the assignment would break other uses of __all__ (e.g. + # `__all__.append(...)`), so keep it if referenced anywhere else. + referenced_elsewhere = any( + isinstance(name, ast.Name) and name.id == '__all__' and isinstance(name.ctx, ast.Load) + for stmt in node.body if stmt not in assigns + for name in ast.walk(stmt) + ) + if referenced_elsewhere: + return node + + node.body = [stmt for stmt in node.body if stmt not in assigns] return node diff --git a/src/terser/_pipeline/transforms/remove_annotations.py b/src/terser/_pipeline/transforms/remove_annotations.py index 5db58c9..c7cbc04 100644 --- a/src/terser/_pipeline/transforms/remove_annotations.py +++ b/src/terser/_pipeline/transforms/remove_annotations.py @@ -2,8 +2,21 @@ from terser.ast import ast, ref from terser.config import RemoveAnnotationOptions, TransformConfig +from terser.utils.imports import qualified_name from ._suite import SuiteTransformer, TransformerFlag +_ANNOTATED_NAMES = ("typing.Annotated", "typing_extensions.Annotated") + + +def _is_annotated(annotation: ast.expr | None) -> bool: + """ + `Annotated[X, ...]` metadata is often consumed at runtime (e.g. pydantic `Field(...)`, FastAPI + dependencies) - a bare signature/attribute type can't express that, so stripping it risks + losing information the annotation itself never encoded. Bare string (forward-ref) annotations + can't be `Annotated[...]` at the AST level, so they always fall through and get stripped. + """ + return isinstance(annotation, ast.Subscript) and qualified_name(annotation.value) in _ANNOTATED_NAMES + class RemoveAnnotations(SuiteTransformer): """ @@ -28,7 +41,7 @@ def visit_FunctionDef(self, node): if hasattr(node, 'type_params') and node.type_params is not None: node.type_params = [self.visit(t) for t in node.type_params] - if hasattr(node, 'returns') and self._options.remove_return_annotations: + if hasattr(node, 'returns') and self._options.remove_return_annotations and not _is_annotated(node.returns): node.returns = None return node @@ -46,14 +59,14 @@ def visit_arguments(self, node): node.kwonlyargs = [self.visit_arg(a) for a in node.kwonlyargs] if hasattr(node, 'varargannotation'): - if self._options.remove_argument_annotations: + if self._options.remove_argument_annotations and not _is_annotated(node.varargannotation): node.varargannotation = None else: if node.vararg: node.vararg = self.visit_arg(node.vararg) if hasattr(node, 'kwargannotation'): - if self._options.remove_argument_annotations: + if self._options.remove_argument_annotations and not _is_annotated(node.kwargannotation): node.kwargannotation = None else: if node.kwarg: @@ -62,7 +75,7 @@ def visit_arguments(self, node): return node def visit_arg(self, node): - if self._options.remove_argument_annotations: + if self._options.remove_argument_annotations and not _is_annotated(node.annotation): node.annotation = None return node @@ -112,7 +125,7 @@ def is_typing_sensitive(node_ref): if not self._options.remove_variable_annotations: return node - if is_dataclass_field(node_ref) or is_typing_sensitive(node_ref): + if is_dataclass_field(node_ref) or is_typing_sensitive(node_ref) or _is_annotated(node.annotation): return node elif node.value: return self.add_child(ast.Assign([node.target], node.value), parent=node_ref.parent, namespace=node_ref.namespace) diff --git a/src/terser/config.py b/src/terser/config.py index d434721..073659d 100644 --- a/src/terser/config.py +++ b/src/terser/config.py @@ -106,8 +106,10 @@ class TransformConfig: remove_generics: bool = True """Remove bare (non-parametrized) `Generic` base classes""" - remove_typing_classes: bool = True - """Remove bare `Protocol` base classes (unless `@typing.runtime_checkable`)""" + remove_typing_classes: bool = False + """Remove bare `Protocol` base classes (unless `@typing.runtime_checkable`). Unsafe across module + boundaries: a stripped class loses Protocol semantics even where another module subclasses it + together with `Protocol[...]`, which raises `TypeError` at class-definition time.""" convert_typing_constructors: bool = True """Convert simple `NamedTuple`/`TypedDict` class definitions to `namedtuple`/`dict` constructors""" @@ -125,5 +127,8 @@ class TransformConfig: convert_posargs: bool = True """Convert positional-only arguments to normal arguments""" - remove_dunder_all: bool = True - """Remove the top-level `__all__` assignment""" + remove_dunder_all: bool = False + """Remove the top-level `__all__` assignment. Unsafe across module boundaries: another + module doing `from this_module import *` relies on `__all__` (falling back to "no names" + when every top-level name is prefixed with an underscore), which a per-module pass run + before project-wide linking has no way to see.""" diff --git a/src/terser/project.py b/src/terser/project.py index ac1f4f4..6044891 100644 --- a/src/terser/project.py +++ b/src/terser/project.py @@ -215,7 +215,8 @@ async def module(node: ast.Module, /): await _write_async(dest, source, limiter=self.__limiter) async def binary(path: Path, /): - assert self.__output + if self.__output is None: + return root = None for r in self.__pp.roots: From 41097cab9e7f086f3d332528a4ce7f830884305e Mon Sep 17 00:00:00 2001 From: Claude Sonnet 5 Date: Sun, 26 Jul 2026 16:40:33 +0000 Subject: [PATCH 12/14] fix(mangler): keep the local binding name on renamed from-import links When a `from x import y` origin binding gets renamed, only the imported name was being synced (`alias_node.name`) - the local binding's own name (which may be preserved via `--preserve-globals`, e.g. `app:app`) was dropped whenever it didn't happen to match the origin's new name, since the old `asname == name` check compared against the already-updated `name`. A re-export like `app/__init__.py: from .main import app` would lose the `app` name entirely once `main.app` got mangled to something else, instead of becoming `from .main import as app`. --- src/terser/_pipeline/mangler/_globals.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/terser/_pipeline/mangler/_globals.py b/src/terser/_pipeline/mangler/_globals.py index 3f9d45b..3e3cb12 100644 --- a/src/terser/_pipeline/mangler/_globals.py +++ b/src/terser/_pipeline/mangler/_globals.py @@ -20,9 +20,10 @@ def _disallow(project: dict[str, ModuleRef], rename_globals: bool, preserved: di def _from_import_links(project: dict[str, ModuleRef]): """ - (alias node, origin binding) pairs for `from x import y [as z]`, where `y` is a name bound - in `x` (rather than a submodule of `x`) - once `y` is renamed, the alias's imported name - must follow, independently of whatever local name `z`/`y` mangles to in the importer. + (alias node, local binding, origin binding) triples for `from x import y [as z]`, where `y` + is a name bound in `x` (rather than a submodule of `x`) - once `y` is renamed, the alias's + imported name must follow, independently of whatever local name `z`/`y` mangles to in the + importer. """ links = [] @@ -39,7 +40,7 @@ def _from_import_links(project: dict[str, ModuleRef]): origin = next((b for b in binding.target.bindings if b.name == binding.target_name), None) if origin is not None: - links.append((binding.node, origin)) + links.append((binding.node, binding, origin)) return links @@ -144,10 +145,9 @@ def mangle_globals(project: dict[str, ModuleRef], rename_globals: bool = False, for namespace, binding in pairs: assigner.assign(namespace, binding) - for alias_node, origin in from_import_links: + for alias_node, local, origin in from_import_links: alias_node.name = origin.name - if alias_node.asname == alias_node.name: - alias_node.asname = None + alias_node.asname = local.name if local.name != alias_node.name else None for attribute_node, origin in attribute_links: attribute_node.attr = origin.name From 07da007c7668113589665f44cf0ee33e902efef2 Mon Sep 17 00:00:00 2001 From: Alpha Date: Sun, 26 Jul 2026 16:41:45 +0000 Subject: [PATCH 13/14] chore: moved some local test files into ignored folder --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a12d8b1..1c66412 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -test.py +/work/ ### Python From f455461fa1b202309918c9f7ff31e18cc8cdba89 Mon Sep 17 00:00:00 2001 From: Claude Sonnet 5 Date: Mon, 27 Jul 2026 01:51:11 +0000 Subject: [PATCH 14/14] fix: address batch of issues.md findings across transforms/mangler/CLI Feature work: - hint_modules config + terser/utils/hints.py: recognize terser_hints decorators from additional user-configured modules, not just terser_hints itself. - terser_hints.constant / preserve_annotations markers, plus ApplyConstantDecorator transform that un-sugars `@lambda _: _()` / `@terser_hints.constant` into a plain call+rebind for later passes to collapse. - constant_folding: fold `if False and ...`-style BoolOp chains, bare `TYPE_CHECKING` (not just `typing.TYPE_CHECKING`), and `sys.version_info (...)` comparisons against a new `target_version` config option. - remove_generics: also strip unused PEP 695 `class Foo[T]` type params, guarded against exported classes and any use of the param name in the class body. - remove_dunder_all_modules: glob-pattern whitelist for removing `__all__` per module, instead of the all-or-nothing `remove_dunder_all` switch. - token_printer: prefer a raw string literal (`r"..."`) over `repr()`'s escaped form when it's shorter and representable. - __import__("mod") / __import__("mod").attr now resolve through a real DynamicImportBinding (terser/_pipeline/resolver/dynamic_import.py), not ad-hoc pattern matching - qualified_name, renaming, and typing-aware transforms all see these the same as a real import. Designed to extend to future forms (__lazy_import__, importlib.import_module) in one place. Correctness fixes found along the way: - CLI: TerserParsedArguments.from_argparse silently dropped most TransformConfig fields (respect_all, remove_dunder_all, passes, contracts, ...); type=bool made `--flag False` parse as True; a Literal[...] field's own alias was passed as `type=`, breaking `--optimize`. All fixed. - Single-file `terser.minify()` defaulted `link_imports=False` and `DummySpec.resolve()` unconditionally raised, so every qualified_name- based check (typing decorator/class removal, TYPE_CHECKING folding, ...) silently no-op'd outside project mode. - mark_exports treated any plain import as part of a module's public surface when there's no `__all__`; now only an explicit `import x as x` re-export counts, matching ruff/pyflakes F401. - resolve_all only recognized `__all__ = [...]`, not `__all__ = (...)`, so tuple-style `__all__` (e.g. pydantic's) parsed as an empty export set. - qualified_name used a binding's local (possibly aliased/mangled) name as the remote symbol name; added ImportBinding.remote_name so `from x import y as z` resolves as `x.y`, not `x.z`. - Binding gained remove_reference(), used when a transform deletes the node that made a reference (e.g. a stripped @typing.override) so stale entries don't keep bindings looking used. - apply_pass's per-transform skip check only looked at earlier-positioned transforms in the current pass, missing changes a later-positioned transform made *last* pass - added `cache.previous_passes` so the round-robin dependency is checked in both directions. - ConvertToLambda/ConvertToInline used `ref(parent).namespace` for a new node's scope, which is one level too far up when `parent` is itself a scope node - fixed to reuse `ref(node).namespace` from the node being replaced. This was a real correctness bug (a mangler namespace-chain walk could loop forever, or definition and use sites could get renamed inconsistently, e.g. `def foo(): ...; return foo` -> `def foo(): ...; return B` with `foo` never renamed at its own definition). - ConvertToLambda/ConvertToInline also fully re-registered reused subtrees via add_child, double-binding references (inflating a builtin's apparent reference count) or, at FLAGS=0, prematurely creating placeholder UnresolvedBindings that permanently squatted a name. Both now do a lightweight reparent instead for reused nodes. - resolver/binding.py: `Foo.__some_method`-style Python-mangled private names in a class body are no longer blanket-disallowed from renaming (Python's own compiler already makes them unreachable under their literal spelling from outside the class). - NameBinding.should_rename used `<=`, renaming on a byte-cost tie for no actual gain; changed to `<` (strict improvement required). - mangle_locals pre-reserves a global's own name to protect locals from colliding with it; mangle_globals's later rename check for that same binding then saw its own reservation as "taken by someone else" and force-renamed it regardless of cost. NameAssigner.assign now clears a binding's self-reservation before checking availability. - ImportBinding.rename()'s `ast.arguments` case checked `node.vararg` twice (copy-paste) instead of `node.kwarg` for `**kwargs`, so keyword- varargs parameters were never actually renamed. - constant_folding.visit_Name folded any qualified-name match regardless of ast context, including a Store target (e.g. the `x` in `x = __import__("typing").TYPE_CHECKING`) - restricted to Load. Co-Authored-By: Claude Sonnet 5 --- src/terser/_minify.py | 2 +- src/terser/_pipeline/mangler/_locals.py | 11 +++ src/terser/_pipeline/printer/token_printer.py | 28 +++++++ src/terser/_pipeline/resolver/binder/_all.py | 2 +- src/terser/_pipeline/resolver/binder/_bind.py | 8 +- .../resolver/binder/_mark_exports.py | 27 +++++- src/terser/_pipeline/resolver/binding.py | 63 +++++++++++++- .../_pipeline/resolver/dynamic_import.py | 43 ++++++++++ src/terser/_pipeline/resolver/resolver.py | 32 ++++++- src/terser/_pipeline/resolver/util.py | 13 +++ src/terser/_pipeline/transforms/__init__.py | 4 +- src/terser/_pipeline/transforms/_suite.py | 22 +++-- .../transforms/apply_constant_decorator.py | 68 +++++++++++++++ .../_pipeline/transforms/constant_folding.py | 84 ++++++++++++++++++- .../_pipeline/transforms/convert_to_inline.py | 40 ++++++++- .../_pipeline/transforms/convert_to_lambda.py | 65 +++++++++++++- src/terser/_pipeline/transforms/remove_all.py | 18 +++- .../transforms/remove_annotations.py | 9 +- .../_pipeline/transforms/remove_docstrings.py | 4 +- .../_pipeline/transforms/remove_generics.py | 30 ++++++- .../transforms/remove_typing_decorators.py | 26 +++++- src/terser/ast/ref/_module/_spec.py | 8 +- src/terser/cli/_argparse.py | 27 ++++-- src/terser/cli/_argv.py | 32 ++++++- src/terser/config.py | 28 ++++++- src/terser/utils/hints.py | 13 +++ src/terser/utils/imports.py | 18 ++-- src/terser_hints/__init__.py | 21 ++++- 28 files changed, 683 insertions(+), 63 deletions(-) create mode 100644 src/terser/_pipeline/resolver/dynamic_import.py create mode 100644 src/terser/_pipeline/transforms/apply_constant_decorator.py create mode 100644 src/terser/utils/hints.py diff --git a/src/terser/_minify.py b/src/terser/_minify.py index 049c367..f96731b 100644 --- a/src/terser/_minify.py +++ b/src/terser/_minify.py @@ -53,7 +53,7 @@ def minify( /, config: TransformConfig, *, - link_imports: bool = False, + link_imports: bool = True, strict: bool = False, defines: dict[str, bool] | None = None, rename: bool = True, diff --git a/src/terser/_pipeline/mangler/_locals.py b/src/terser/_pipeline/mangler/_locals.py index c30927b..f1ee6b4 100644 --- a/src/terser/_pipeline/mangler/_locals.py +++ b/src/terser/_pipeline/mangler/_locals.py @@ -234,6 +234,17 @@ def assign(self, namespace, binding, *, prefix=''): scope = reservation_scope(namespace, binding) if binding.allow_rename: + # A binding may have already reserved its own current name in an earlier, + # separate pass (e.g. mangle_locals reserving every global's name so local + # mangling doesn't shadow it, before mangle_globals gets a turn at the same + # binding) - undo that self-reservation before checking availability, or + # `should_rename` sees its own name as "already taken" and force-renames it + # even when keeping it would be shorter. + for ns in scope: + ns_ref = ref(ns) + if hasattr(ns_ref, 'assigned_names'): + ns_ref.assigned_names.discard(binding.name) + name = self.available_name(scope, prefix=prefix) if should_rename(binding, name, scope, self.is_available): diff --git a/src/terser/_pipeline/printer/token_printer.py b/src/terser/_pipeline/printer/token_printer.py index ea2769d..1415e4d 100644 --- a/src/terser/_pipeline/printer/token_printer.py +++ b/src/terser/_pipeline/printer/token_printer.py @@ -3,6 +3,30 @@ from re import compile as _re +def _raw_literal(value: str) -> str | None: + """ + Render `value` as a raw string literal (`r"..."`/`r'...'`), or `None` if it can't be + one - `repr()` always escapes backslashes, which is longer than the source for + backslash-heavy strings like regex patterns (`r"some\\.raw\\.strings"`). + + A raw string can't represent: non-printable characters (no escape sequences at all + in raw mode), a trailing odd run of backslashes (would escape the closing quote), or + a value containing both quote characters (nothing left to delimit it with). + """ + if not value.isprintable(): + return None + + trailing_backslashes = len(value) - len(value.rstrip('\\')) + if trailing_backslashes % 2 == 1: + return None + + for quote in ("'", '"'): + if quote not in value: + return f"r{quote}{value}{quote}" + + return None + + class TokenTypes(IntEnum): NoToken = 0 Identifier = 1 @@ -143,6 +167,10 @@ def stringliteral(self, value): """Add a string literal to the output code.""" s = repr(value) + raw = _raw_literal(value) + if raw is not None and len(raw) < len(s): + s = raw + if len(s) > 0 and s[0].isalpha() and self.previous_token in [TokenTypes.Identifier, TokenTypes.Keyword, TokenTypes.SoftKeyword]: self.delimiter(' ') diff --git a/src/terser/_pipeline/resolver/binder/_all.py b/src/terser/_pipeline/resolver/binder/_all.py index 7bc5e9c..96c69fd 100644 --- a/src/terser/_pipeline/resolver/binder/_all.py +++ b/src/terser/_pipeline/resolver/binder/_all.py @@ -25,7 +25,7 @@ def resolve_all(module: ast.Module): if not __is_assign(node): continue - if not isinstance(node.value, ast.List): + if not isinstance(node.value, (ast.List, ast.Tuple)): continue for el in node.value.elts: diff --git a/src/terser/_pipeline/resolver/binder/_bind.py b/src/terser/_pipeline/resolver/binder/_bind.py index 8c0c673..86d3b80 100644 --- a/src/terser/_pipeline/resolver/binder/_bind.py +++ b/src/terser/_pipeline/resolver/binder/_bind.py @@ -3,7 +3,7 @@ from terser.ast import ModuleRef, ast, ref from ..binding import Binding, BuiltinBinding, UnresolvedBinding -from ..util import scope_ref_global, scope_ref_nonlocal +from ..util import is_python_mangled_private, scope_ref_global, scope_ref_nonlocal from ...parser._scope import ScopeResolver if TYPE_CHECKING: @@ -42,8 +42,10 @@ def __get_binding(name: str, namespace_ref: ScopedNode) -> Binding: def __attr_get_binding(name: str, namespace: ScopedNode) -> Binding: binding = __get_binding(name, namespace) - if isinstance(namespace.ast, ast.ClassDef): - # This name will become an attribute of a class, so it can't be renamed + if isinstance(namespace.ast, ast.ClassDef) and not is_python_mangled_private(name): + # This name will become an attribute of a class, so it can't be renamed - + # unless Python's own compiler already private-mangles it (`__foo`), in which + # case it's already unreachable from outside under its literal spelling. binding.disallow_rename() return binding diff --git a/src/terser/_pipeline/resolver/binder/_mark_exports.py b/src/terser/_pipeline/resolver/binder/_mark_exports.py index a1b508e..fe51844 100644 --- a/src/terser/_pipeline/resolver/binder/_mark_exports.py +++ b/src/terser/_pipeline/resolver/binder/_mark_exports.py @@ -1,11 +1,26 @@ from typing import TYPE_CHECKING -from terser.ast import ref +from terser.ast import ast, ref +from ..binding import ImportBinding if TYPE_CHECKING: from ast import Module +def _is_intentional_reexport(binding) -> bool: + """ + Without an explicit `__all__`, a locally-defined name is implicitly part of the module's + public interface - but a plain import isn't: `from typing import override` used only for + a decorator isn't "re-exporting `override`" by convention (matching ruff/pyflakes F401), + only the explicit `import x as x` / `from y import x as x` idiom is. + """ + if not isinstance(binding, ImportBinding): + return True + + node = binding.node + return isinstance(node, ast.alias) and node.asname == node.name + + def mark_exports(module: Module) -> None: """ Flag the module-level bindings that make up `module_ref`'s public interface - importable via @@ -17,9 +32,13 @@ def mark_exports(module: Module) -> None: """ module_ref = ref(module) - exported_names: set[str] = module_ref.all or { - name for binding in module_ref.bindings if (name := binding.name) and not name.startswith('__') - } + if module_ref.all is not None: + exported_names: set[str] = module_ref.all + else: + exported_names = { + name for binding in module_ref.bindings + if (name := binding.name) and not name.startswith('__') and _is_intentional_reexport(binding) + } for binding in module_ref.bindings: if binding.name in exported_names: diff --git a/src/terser/_pipeline/resolver/binding.py b/src/terser/_pipeline/resolver/binding.py index f042bd8..b4e0c56 100644 --- a/src/terser/_pipeline/resolver/binding.py +++ b/src/terser/_pipeline/resolver/binding.py @@ -273,6 +273,16 @@ def add_reference(self, node: ast.AST, allow_rename: bool = True, reserved: str if reserved is not None: self._reserved = reserved + def remove_reference(self, node: ast.AST): + """ + Drop a reference from this binding, e.g. when a transform deletes the node that + made it (a stripped decorator, a removed branch) - otherwise the stale entry keeps + the binding looking used to reference-count-based checks (unused-import cleanup, + safe-to-drop-definition checks) even after nothing in the tree points to it anymore. + """ + + self._references.remove(node) + @abstractmethod def should_rename(self, new_name: str) -> bool: """ @@ -330,7 +340,10 @@ def should_rename(self, new_name: str): additional_bytes = self.additional_byte_cost() rename_cost = (old_mentions * len(self.name)) + (new_mentions * len(new_name)) + additional_bytes - return rename_cost <= current_cost + # Strict improvement only - a tie is no gain, and leaving the original name alone + # keeps it free for whatever *other* binding would otherwise need to fall back to a + # longer candidate to avoid colliding with a pointless same-length rename. + return rename_cost < current_cost @override def disallow_rename(self): @@ -369,7 +382,7 @@ def rename(self, new_name: str): if (vararg := node.vararg) and (vararg.arg == self.name) and not getattr(node, "vararg_renamed", False): vararg.arg = new_name setattr(node, "vararg_renamed", True) - if (kwarg := node.vararg) and (kwarg.arg == self.name) and not getattr(node, "kwarg_renamed", False): + if (kwarg := node.kwarg) and (kwarg.arg == self.name) and not getattr(node, "kwarg_renamed", False): kwarg.arg = new_name setattr(node, "kwarg_renamed", True) @@ -451,6 +464,52 @@ def source_module(self) -> str | None: ref = self._module_ref.import_targets.get(self) return ref.path if ref is not None else None + @property + def remote_name(self) -> str | None: + """ + The name this binding refers to *in its source module*, when that's not the same as + the local (possibly aliased, possibly later mangled) `.name` - e.g. `"override"` for + `from typing import override as ov`. `None` for a plain `import x [as y]` (the binding + names the module itself, not a symbol within it) or a wildcard-derived binding, where + `.name` is already the right thing to qualify `source_module` with. + """ + if isinstance(self.node, ast.alias) and isinstance(ref(self.node).parent, ast.ImportFrom): + return self.node.name + + return None + + +class DynamicImportBinding(ImportBinding): + """ + An `ImportBinding` synthesized from a dynamic-import expression - `__import__("mod")` or + `__import__("mod").attr` - assigned to a name, rather than a literal `import`/`from import` + statement. Lets `qualified_name` and the project-wide mangler treat these the same as a + real import: renamed/tracked consistently, and recognized by typing-aware transforms + (`TYPE_CHECKING` folding, `@typing.override` stripping, etc). + + `target`/`target_name` (cross-module linking) are never populated here - resolving which + project module a dynamic import call refers to isn't part of `resolve_imports`'s static + import graph, so cross-module linking is out of scope for these for now. + + See `terser._pipeline.resolver.dynamic_import` for the recognizer this is built from - + add new dynamic-import forms there, not here. + """ + + def __init__(self, name, node, module_ref: ModuleRef, source_module: str, remote_name: str | None, *args, **kwargs): + super().__init__(name, node, module_ref, *args, **kwargs) + self._source_module = source_module + self._remote_name = remote_name + + @override + @property + def source_module(self) -> str | None: + return self._source_module + + @override + @property + def remote_name(self) -> str | None: + return self._remote_name + class UnresolvedBinding(NameBinding): """ diff --git a/src/terser/_pipeline/resolver/dynamic_import.py b/src/terser/_pipeline/resolver/dynamic_import.py new file mode 100644 index 0000000..fb59dfc --- /dev/null +++ b/src/terser/_pipeline/resolver/dynamic_import.py @@ -0,0 +1,43 @@ +from terser.ast import ast + +# Callables recognized as dynamic-import forms. Extend here (not at each call site) when +# adding new forms - e.g. a future `__lazy_import__`, or `importlib.import_module`. +_DYNAMIC_IMPORT_CALLEES = ('__import__',) + + +def match_dynamic_import_call(node: ast.expr) -> str | None: + """ + Recognize a dynamic-import call (`__import__("mod")`), returning the literal module + name, or `None` if `node` isn't one. + """ + if not isinstance(node, ast.Call) or len(node.args) != 1 or node.keywords: + return None + + if not isinstance(node.func, ast.Name) or node.func.id not in _DYNAMIC_IMPORT_CALLEES: + return None + + arg = node.args[0] + if not isinstance(arg, ast.Constant) or not isinstance(arg.value, str): + return None + + return arg.value + + +def match_dynamic_import_value(node: ast.expr) -> tuple[str, str | None] | None: + """ + Recognize an assignable dynamic-import expression - `__import__("mod")` or + `__import__("mod").attr` - returning `(source_module, remote_name)`, where + `remote_name` is `None` for the bare (module-only) form. + """ + remote_name = None + call = node + + if isinstance(call, ast.Attribute): + remote_name = call.attr + call = call.value + + source_module = match_dynamic_import_call(call) + if source_module is None: + return None + + return source_module, remote_name diff --git a/src/terser/_pipeline/resolver/resolver.py b/src/terser/_pipeline/resolver/resolver.py index ef847d5..8a2ec1b 100644 --- a/src/terser/_pipeline/resolver/resolver.py +++ b/src/terser/_pipeline/resolver/resolver.py @@ -2,8 +2,9 @@ from typing import TYPE_CHECKING, override from terser.ast import NodeVisitor, ast, ref -from .binding import Binding, ImportBinding, NameBinding -from .util import arg_rename_in_place, scope_ref_global +from .binding import Binding, DynamicImportBinding, ImportBinding, NameBinding +from .dynamic_import import match_dynamic_import_value +from .util import arg_rename_in_place, is_python_mangled_private, scope_ref_global if TYPE_CHECKING: from collections.abc import Callable @@ -71,8 +72,10 @@ def __get_binding(self, name: str, namespace: ContainsScope, factory: Callable[[ # This is actually a syntax error - but we want the same syntax error after minifying! binding.disallow_rename() - if isinstance(namespace, ast.ClassDef): - # This name will become an attribute of the class, so it can't be renamed + if isinstance(namespace, ast.ClassDef) and not is_python_mangled_private(name): + # This name will become an attribute of the class, so it can't be renamed - + # unless Python's own compiler already private-mangles it (`__foo`), in which + # case it's already unreachable from outside under its literal spelling. binding.disallow_rename() return binding @@ -88,6 +91,27 @@ def visit_Name(self, node: ast.Name): if isinstance(node.ctx, (ast.Store, ast.Del)): self.__get_binding(node.id, namespace).add_reference(node) + @override + def visit_Assign(self, node: ast.Assign): + match = None + target = node.targets[0] if len(node.targets) == 1 else None + if isinstance(target, ast.Name): + match = match_dynamic_import_value(node.value) + + if match is None: + self.generic_visit(node) + return + + source_module, remote_name = match + namespace = ref(target).namespace + assert isinstance(target, ast.Name) + + if target.id not in ref(namespace).nonlocals: + factory = lambda name: DynamicImportBinding(name, target, self.module_ref, source_module, remote_name) + self.__get_binding(target.id, namespace, factory).add_reference(target) + + self.visit(node.value) + @override def visit_ClassDef(self, node: ast.ClassDef): namespace = ref(node).namespace diff --git a/src/terser/_pipeline/resolver/util.py b/src/terser/_pipeline/resolver/util.py index f168709..a8d6e98 100644 --- a/src/terser/_pipeline/resolver/util.py +++ b/src/terser/_pipeline/resolver/util.py @@ -35,6 +35,19 @@ def scope_ref_nonlocal(node: ast.AST) -> ScopedNode: return ref(namespace) +def is_python_mangled_private(name: str) -> bool: + """ + Does Python's own compiler already private-name-mangle this identifier + + A class-body name with at least two leading underscores and at most one trailing + underscore (e.g. `__foo`, but not `__foo__`) gets rewritten by the compiler to + `_ClassName__foo` wherever it's used inside that class - external code can't reach it + under its literal spelling without already knowing the mangled form, so terser + renaming it further doesn't lose anything Python wasn't already hiding. + """ + return name.startswith('__') and not name.endswith('__') + + def arg_rename_in_place(node: ast.AST, /) -> bool: """ Can this argument node by safely renamed diff --git a/src/terser/_pipeline/transforms/__init__.py b/src/terser/_pipeline/transforms/__init__.py index 7e2fc8f..65f9691 100644 --- a/src/terser/_pipeline/transforms/__init__.py +++ b/src/terser/_pipeline/transforms/__init__.py @@ -28,6 +28,7 @@ from .remove_exception_brackets import RemoveExceptionBrackets from .remove_posargs import RemovePosArgs from .remove_all import RemoveAll +from .apply_constant_decorator import ApplyConstantDecorator __transforms__ = [ @@ -44,11 +45,12 @@ RemoveExplicitReturnNone, ConvertEarlyExits, ConvertToInline, - ConvertToLambda, # FLAGS = REQUIRES_IMPORT_RESOLVE Contracts, + ApplyConstantDecorator, RemoveAnnotations, + ConvertToLambda, RemoveDummyAssignments, RemoveDocstrings, CleanupLocalImports, diff --git a/src/terser/_pipeline/transforms/_suite.py b/src/terser/_pipeline/transforms/_suite.py index 79b79af..7f1a0b3 100644 --- a/src/terser/_pipeline/transforms/_suite.py +++ b/src/terser/_pipeline/transforms/_suite.py @@ -34,6 +34,7 @@ class TransformCache: transforms: list[type[SuiteTransformer]] = field(default_factory=list) passes: dict[type[SuiteTransformer], bool] = field(default_factory=dict) + previous_passes: dict[type[SuiteTransformer], bool] = field(default_factory=dict) def apply_pass(cache: TransformCache, module: ast.Module, transform_types: Iterable[type[SuiteTransformer]], flags_max: TransformerFlag | int) -> ast.Module: @@ -43,10 +44,14 @@ def apply_pass(cache: TransformCache, module: ast.Module, transform_types: Itera Callers should loop this until `not any(cache.passes.values())` (nothing changed this pass) or `config.passes` is reached - `SuiteTransformer.__new__` - uses `cache.passes` to skip re-running a transform when nothing earlier in - `cache.transforms` changed since the last pass. + uses `cache.passes`/`cache.previous_passes` to skip re-running a transform when + it has no new work: nothing earlier in `cache.transforms` changed the tree so far + this pass, and nothing from its own position onward changed it last pass either + (a change there hasn't been seen by this transform yet, since this round-robin + sweep hasn't reached it again). """ cache.transforms = [t for t in transform_types if t.is_enabled(cache.config) and t.FLAGS <= flags_max] + cache.previous_passes = dict(cache.passes) for transform in cache.transforms: before = ast.dump(module) @@ -79,10 +84,15 @@ def __new__(cls, ctx: TransformConfig | TransformCache, /) -> Self: return obj assert not set(ctx.transforms).difference(set(ctx.passes.keys())) - for i in range(ctx.transforms.index(cls)): - if ctx.passes[ctx.transforms[i]]: - break - else: + idx = ctx.transforms.index(cls) + + # Something before me changed the tree already this pass, or something from my own + # position onward changed it last pass (a round-robin sweep, so a change there hasn't + # reached me again yet) - either way, there may be new work for me to do. + changed_before_this_pass = any(ctx.passes[t] for t in ctx.transforms[:idx]) + changed_from_here_last_pass = any(ctx.previous_passes.get(t, True) for t in ctx.transforms[idx:]) + + if not changed_before_this_pass and not changed_from_here_last_pass: return lambda _: _ # type: ignore[ty:invalid-return-type] obj = super().__new__(cls) diff --git a/src/terser/_pipeline/transforms/apply_constant_decorator.py b/src/terser/_pipeline/transforms/apply_constant_decorator.py new file mode 100644 index 0000000..995cbe0 --- /dev/null +++ b/src/terser/_pipeline/transforms/apply_constant_decorator.py @@ -0,0 +1,68 @@ +from typing import override + +from terser.ast import ast +from terser.config import TransformConfig +from terser.utils.hints import is_hinted +from terser.utils.imports import qualified_name +from ._suite import SuiteTransformer, TransformerFlag + + +def _is_invoke_lambda(decorator: ast.expr) -> bool: + """ + Matches the `@lambda _: _()` idiom: a single-arg lambda whose body is a no-arg call + of that same argument - used as a decorator to immediately invoke a `def`, since + Python has no IIFE syntax for function statements. + """ + if not isinstance(decorator, ast.Lambda): + return False + + args = decorator.args + if args.posonlyargs or args.kwonlyargs or args.vararg or args.kwarg or len(args.args) != 1: + return False + + body = decorator.body + return ( + isinstance(body, ast.Call) and not body.args and not body.keywords + and isinstance(body.func, ast.Name) and body.func.id == args.args[0].arg + ) + + +class ApplyConstantDecorator(SuiteTransformer): + """ + Un-sugar the `@lambda _: _()` / `@terser_hints.constant` "call this function once and + rebind its name to the result" marker into a plain call and rebind, so later passes + (`ConvertToLambda`, `UnfoldIIFE`) can collapse it further. + """ + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE + + @override + @classmethod + def is_enabled(cls, config: TransformConfig, /) -> bool: + return config.unfold_iife_lambdas + + def _is_marked(self, decorator: ast.expr) -> bool: + return _is_invoke_lambda(decorator) or qualified_name(decorator) in ( + {"terser_hints.constant"} | {f"{m}.constant" for m in self._config.hint_modules} + ) + + @override + def suite(self, node_list, parent): + result = [] + for node in node_list: + node = self.visit(node) + + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and any(self._is_marked(d) for d in node.decorator_list): + node.decorator_list = [d for d in node.decorator_list if not self._is_marked(d)] + result.append(node) + result.append(self.add_child( + ast.Assign( + targets=[ast.Name(id=node.name, ctx=ast.Store())], + value=ast.Call(func=ast.Name(id=node.name, ctx=ast.Load()), args=[], keywords=[]), + ), + parent=parent, + )) + continue + + result.append(node) + + return result diff --git a/src/terser/_pipeline/transforms/constant_folding.py b/src/terser/_pipeline/transforms/constant_folding.py index 7eee9cb..3b9d145 100644 --- a/src/terser/_pipeline/transforms/constant_folding.py +++ b/src/terser/_pipeline/transforms/constant_folding.py @@ -1,4 +1,5 @@ import math +import operator from typing import TYPE_CHECKING, override from terser.ast import ast, compare_ast, is_constant_node, ref @@ -25,6 +26,9 @@ def _is_unshadowed_builtin(node, name: str) -> bool: return isinstance(binding, BuiltinBinding) and binding.name == name and not binding.is_redefined() +_TYPE_CHECKING_NAMES = ('typing.TYPE_CHECKING', 'typing_extensions.TYPE_CHECKING') + + def is_provably_bool(node) -> bool: """ Check if a node's value is guaranteed to be a bool, syntactically. @@ -168,10 +172,44 @@ def visit_UnaryOp(self, node): return self.fold(node) + def _fold_version_info(self, node): + if self._config.target_version is None or len(node.ops) != 1: + return None + + ops = { + ast.Lt: operator.lt, ast.LtE: operator.le, ast.Gt: operator.gt, + ast.GtE: operator.ge, ast.Eq: operator.eq, ast.NotEq: operator.ne, + } + op = ops.get(type(node.ops[0])) + if op is None: + return None + + left, right = node.left, node.comparators[0] + swapped = qualified_name(left) != 'sys.version_info' + if swapped: + left, right = right, left + + if qualified_name(left) != 'sys.version_info' or not isinstance(right, ast.Tuple): + return None + + literal: list[int] = [] + for elt in right.elts: + if not is_constant_node(elt, ast.Num) or not isinstance(elt.value, int): + return None + literal.append(elt.value) + + target = tuple(self._config.target_version[:len(literal)]) + a, b = (tuple(literal), target) if swapped else (target, tuple(literal)) + return ast.NameConstant(value=op(a, b)) + def visit_Compare(self, node): node.left = self.visit(node.left) node.comparators = [self.visit(c) for c in node.comparators] + if (new_node := self._fold_version_info(node)) is not None: + node_ref = ref(node) + return self.add_child(new_node, node_ref.parent, node_ref.namespace) + if len(node.ops) != 1 or not isinstance(node.ops[0], (ast.Eq, ast.NotEq, ast.Is, ast.IsNot)): return node @@ -199,23 +237,63 @@ def visit_Compare(self, node): return self.add_child(new_node, node_ref.parent, node_ref.namespace) def visit_Name(self, node): - if node.id != '__debug__' or not _is_unshadowed_builtin(node, '__debug__'): + if not isinstance(node.ctx, ast.Load): + # a Store/Del context is the binding's own definition site, not a usage to + # fold - e.g. the `x` in `x = __import__("typing").TYPE_CHECKING` itself + return node + + if node.id == '__debug__' and _is_unshadowed_builtin(node, '__debug__'): + new_node = ast.NameConstant(value=self._config.optimize < 1) + elif qualified_name(node) in _TYPE_CHECKING_NAMES: + # False at runtime - only ever True for static type checkers + new_node = ast.NameConstant(value=False) + else: return node - new_node = ast.NameConstant(value=self._config.optimize < 1) node_ref = ref(node) return self.add_child(new_node, node_ref.parent, node_ref.namespace) def visit_Attribute(self, node): node.value = self.visit(node.value) - if qualified_name(node) != 'typing.TYPE_CHECKING': + if qualified_name(node) not in _TYPE_CHECKING_NAMES: return node new_node = ast.NameConstant(value=False) node_ref = ref(node) return self.add_child(new_node, node_ref.parent, node_ref.namespace) + def visit_BoolOp(self, node): + node.values = [self.visit(v) for v in node.values] + + literal_types = (ast.Num, ast.NameConstant, ast.Str, ast.Bytes) + stop_at = isinstance(node.op, ast.Or) # `or` short-circuits on truthy, `and` on falsy + + values = node.values + start = 0 + # Leading literals that don't determine the outcome are side-effect free, so + # can be dropped without changing which value the chain evaluates to. + while start < len(values) - 1 and is_constant_node(values[start], literal_types) and bool(values[start].value) != stop_at: + start += 1 + + end = len(values) + for i in range(start, len(values)): + if is_constant_node(values[i], literal_types) and bool(values[i].value) == stop_at: + end = i + 1 # nothing after a short-circuiting literal is ever evaluated + break + + trimmed = values[start:end] + if len(trimmed) == len(values): + return node + + if len(trimmed) == 1: + result = trimmed[0] + ref(result).parent = ref(node).parent + return result + + node.values = trimmed + return node + def visit_Call(self, node): node.func = self.visit(node.func) node.args = [self.visit(a) for a in node.args] diff --git a/src/terser/_pipeline/transforms/convert_to_inline.py b/src/terser/_pipeline/transforms/convert_to_inline.py index 4c83d9e..0c95124 100644 --- a/src/terser/_pipeline/transforms/convert_to_inline.py +++ b/src/terser/_pipeline/transforms/convert_to_inline.py @@ -1,6 +1,7 @@ from typing import override from terser.ast import ast, ref +from terser.ast.ref._node import NodeRef from terser.config import TransformConfig from ._suite import SuiteTransformer @@ -17,6 +18,33 @@ class ConvertToInline(SuiteTransformer): def is_enabled(cls, config: TransformConfig, /) -> bool: return config.convert_to_inline + def _wrap(self, expr: ast.expr, *reused: ast.expr, parent: ast.AST, namespace: ast.AST) -> ast.Expr: + """ + Wrap `expr` (brand new, but holding *reused* subtrees like the `if`'s own test/body) + in an `Expr` statement. + + This runs pre-resolve (`FLAGS = 0`), before names have been bound at all - `add_child` + would work here too, but its full recursive re-registration walk calls `bind_names` on + the reused subtrees as if they were brand new. At this point in the file, a name a reused + subtree references (e.g. a module imported earlier in the file) may not have its real + binding yet, so that premature bind creates a placeholder `UnresolvedBinding` which then + squats the name - once the real declaration is bound later, resolution finds the + placeholder already there and never upgrades it, breaking any later `qualified_name`-based + check on it. Only the genuinely new nodes here (`expr`, the `Expr` wrapper) need a fresh + `NodeRef`; the reused ones just need their `parent` pointer updated. + """ + new_node = ast.Expr(value=expr) + + new_nodes = [new_node, expr, *([expr.op] if isinstance(expr, ast.BoolOp) else [])] + for node in new_nodes: + NodeRef.new(node, parent if node is new_node else expr) + ref(node).namespace = namespace + + for node in reused: + ref(node).parent = expr + + return new_node + @override def visit_If(self, node: ast.If): node: ast.If = self.generic_visit(node) @@ -25,14 +53,18 @@ def visit_If(self, node: ast.If): return node a = node.body[0].value + # The new statement takes `node`'s exact place in the tree, so it belongs to + # exactly the scope `node` already resolved to - no need to recompute it (and + # `ref(parent).namespace` would be wrong: `parent` here is itself a scope node, + # so that would walk one level too far up, to the scope *containing* it). + parent = ref(node).parent + namespace = ref(node).namespace if not node.orelse: - new_node = ast.Expr(value=ast.BoolOp(op=ast.And(), values=[node.test, a])) - return self.add_child(new_node, parent=ref(node).parent) + return self._wrap(ast.BoolOp(op=ast.And(), values=[node.test, a]), node.test, a, parent=parent, namespace=namespace) if len(node.orelse) != 1 or not isinstance(node.orelse[0], ast.Expr): return node b = node.orelse[0].value - new_node = ast.Expr(value=ast.IfExp(test=node.test, body=a, orelse=b)) - return self.add_child(new_node, parent=ref(node).parent) + return self._wrap(ast.IfExp(test=node.test, body=a, orelse=b), node.test, a, b, parent=parent, namespace=namespace) diff --git a/src/terser/_pipeline/transforms/convert_to_lambda.py b/src/terser/_pipeline/transforms/convert_to_lambda.py index d8bae7f..a36712d 100644 --- a/src/terser/_pipeline/transforms/convert_to_lambda.py +++ b/src/terser/_pipeline/transforms/convert_to_lambda.py @@ -1,16 +1,33 @@ from typing import override from terser.ast import ast, ref +from terser.ast.ref._node import NodeRef from terser.config import TransformConfig -from ._suite import SuiteTransformer +from ._suite import SuiteTransformer, TransformerFlag + + +def _is_annotated(args: ast.arguments, returns: ast.expr | None) -> bool: + # A lambda has no syntax for annotations at all - converting a function that still + # has one (e.g. `Annotated[...]` protected from removal) would produce invalid code. + all_args = [*getattr(args, 'posonlyargs', []), *args.args, *args.kwonlyargs] + if args.vararg is not None: + all_args.append(args.vararg) + if args.kwarg is not None: + all_args.append(args.kwarg) + + return returns is not None or any(a.annotation is not None for a in all_args) class ConvertToLambda(SuiteTransformer): """ Convert a single-expression function to a lambda assignment: `def foo(...): return expr` -> `foo = lambda ...: expr` + + Registered after `RemoveAnnotations` (same FLAGS stage) so that by the time this runs, + any annotation that's going to be removed already has been - what's left is exactly + what would survive as invalid syntax on a lambda. """ - FLAGS = 0 + FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE @override @classmethod @@ -24,12 +41,52 @@ def visit_FunctionDef(self, node: ast.FunctionDef): if isinstance(node, ast.AsyncFunctionDef) or node.decorator_list or len(node.body) != 1: return node + if _is_annotated(node.args, getattr(node, 'returns', None)): + return node + stmt = node.body[0] if not isinstance(stmt, ast.Return) or stmt.value is None: # a bare `Expr` body isn't equivalent as a lambda: it always # implicitly returns None, but a lambda returns the expression's value return node + # `node.args`/`body` are reused, already-bound subtrees (this runs post-resolve) - + # a full `add_child` walk would call `bind_names` on them again, double-counting + # every reference they hold (e.g. inflating a builtin's reference count enough to + # make renaming it look worthwhile when it isn't). Only `new_node`/`lam` are + # genuinely new and need a fresh `NodeRef`; the reused ones just need reparenting. body = stmt.value - new_node = ast.Assign(targets=[ast.Name(id=node.name, ctx=ast.Store())], value=ast.Lambda(args=node.args, body=body)) - return self.add_child(new_node, parent=ref(node).parent) + args = node.args + parent = ref(node).parent + # The new statement takes `node`'s exact place in the tree, so it belongs to + # exactly the scope `node` already resolved to - no need to recompute it (and + # `ref(parent).namespace` would be wrong: `parent` here is itself a scope node, + # so that would walk one level too far up, to the scope *containing* it). + namespace = ref(node).namespace + + lam = ast.Lambda(args=args, body=body) + new_node = ast.Assign(targets=[], value=lam) + + for new in (new_node, lam): + NodeRef.new(new, parent if new is new_node else new_node) + ref(new).namespace = namespace + + ref(args).parent = lam + ref(body).parent = lam + + # The new target replaces `node` (the FunctionDef) as what the function's own + # binding points at. `add_child`'s `bind_names` has no case for a plain local + # Store (it only resolves references to *existing* bindings, e.g. nonlocals) - + # it would silently leave this new Name with no binding at all, so the rename + # that already touched every other reference would miss this one. Move the + # reference over directly instead. + target = ast.Name(id=node.name, ctx=ast.Store()) + NodeRef.new(target, new_node) + ref(target).namespace = namespace + + binding = ref(node).binding + binding.remove_reference(node) + binding.add_reference(target) + + new_node.targets = [target] + return new_node diff --git a/src/terser/_pipeline/transforms/remove_all.py b/src/terser/_pipeline/transforms/remove_all.py index 99e1509..ad36546 100644 --- a/src/terser/_pipeline/transforms/remove_all.py +++ b/src/terser/_pipeline/transforms/remove_all.py @@ -1,6 +1,7 @@ +from fnmatch import fnmatch from typing import override -from terser.ast import ast +from terser.ast import ast, ref from terser.config import TransformConfig from ._suite import SuiteTransformer, TransformerFlag @@ -12,19 +13,30 @@ def _is_dunder_all_assign(node) -> bool: ) +def _removal_allowed(module_path: str, config: TransformConfig) -> bool: + if config.remove_dunder_all: + return True + + return any(fnmatch(module_path, pattern) for pattern in config.remove_dunder_all_modules) + + class RemoveAll(SuiteTransformer): """ - Remove the top-level `__all__` assignment + Remove the top-level `__all__` assignment, for modules allowed by `config.remove_dunder_all` + / `config.remove_dunder_all_modules` """ FLAGS = TransformerFlag.INFLUENCES_MANGLING @override @classmethod def is_enabled(cls, config: TransformConfig, /) -> bool: - return config.remove_dunder_all + return config.remove_dunder_all or bool(config.remove_dunder_all_modules) @override def visit_Module(self, node: ast.Module): + if not _removal_allowed(str(ref(node).spec), self._config): + return node + assigns = [stmt for stmt in node.body if _is_dunder_all_assign(stmt)] if not assigns: return node diff --git a/src/terser/_pipeline/transforms/remove_annotations.py b/src/terser/_pipeline/transforms/remove_annotations.py index c7cbc04..3aeae5b 100644 --- a/src/terser/_pipeline/transforms/remove_annotations.py +++ b/src/terser/_pipeline/transforms/remove_annotations.py @@ -2,6 +2,7 @@ from terser.ast import ast, ref from terser.config import RemoveAnnotationOptions, TransformConfig +from terser.utils.hints import is_hinted from terser.utils.imports import qualified_name from ._suite import SuiteTransformer, TransformerFlag @@ -34,14 +35,16 @@ def __init__(self, ctx): self._options = RemoveAnnotationOptions() if isinstance(self._config.remove_annotations, bool) else self._config.remove_annotations def visit_FunctionDef(self, node): - node.args = self.visit_arguments(node.args) + preserved = is_hinted(node.decorator_list, "preserve_annotations", self._config) + + node.args = node.args if preserved else self.visit_arguments(node.args) node.body = self.suite(node.body, parent=node) node.decorator_list = [self.visit(d) for d in node.decorator_list] if hasattr(node, 'type_params') and node.type_params is not None: node.type_params = [self.visit(t) for t in node.type_params] - if hasattr(node, 'returns') and self._options.remove_return_annotations and not _is_annotated(node.returns): + if hasattr(node, 'returns') and self._options.remove_return_annotations and not preserved and not _is_annotated(node.returns): node.returns = None return node @@ -119,7 +122,7 @@ def is_typing_sensitive(node_ref): # is this a class attribute or a variable? node_ref = ref(node) if isinstance(node_ref.parent, ast.ClassDef): - if not self._options.remove_attribute_annotations: + if not self._options.remove_attribute_annotations or is_hinted(node_ref.parent.decorator_list, "preserve_annotations", self._config): return node else: if not self._options.remove_variable_annotations: diff --git a/src/terser/_pipeline/transforms/remove_docstrings.py b/src/terser/_pipeline/transforms/remove_docstrings.py index 53e623c..c41d095 100644 --- a/src/terser/_pipeline/transforms/remove_docstrings.py +++ b/src/terser/_pipeline/transforms/remove_docstrings.py @@ -2,7 +2,7 @@ from terser.ast import ast, is_constant_node from terser.config import RemoveDocstringOptions, TransformConfig -from terser.utils.imports import qualified_name +from terser.utils.hints import is_hinted from ._suite import SuiteTransformer, TransformerFlag @@ -26,7 +26,7 @@ def _has_docstring(self, body) -> bool: return bool(body) and isinstance(body[0], ast.Expr) and is_constant_node(body[0].value, ast.Str) def _preserved(self, decorator_list) -> bool: - return any(qualified_name(d) == "terser_hints.preserve_docstring" for d in decorator_list) + return is_hinted(decorator_list, "preserve_docstring", self._config) def _strip_docstring(self, node): node.body = node.body[1:] diff --git a/src/terser/_pipeline/transforms/remove_generics.py b/src/terser/_pipeline/transforms/remove_generics.py index 07f1350..5ffb533 100644 --- a/src/terser/_pipeline/transforms/remove_generics.py +++ b/src/terser/_pipeline/transforms/remove_generics.py @@ -1,6 +1,6 @@ from typing import override -from terser.ast import ast +from terser.ast import ast, ref from terser.config import TransformConfig from terser.utils.imports import qualified_name from ._suite import SuiteTransformer, TransformerFlag @@ -10,12 +10,27 @@ def _is_bare_generic(base: ast.expr) -> bool: return isinstance(base, (ast.Name, ast.Attribute)) and qualified_name(base) == "typing.Generic" +def _is_subscripted_anywhere(binding) -> bool: + return any(isinstance(r, ast.Name) and isinstance(ref(r).parent, ast.Subscript) for r in binding.references) + + +def _type_param_used_in_body(name: str, body: list[ast.stmt]) -> bool: + # Type params introduce a scope the whole class body (annotations, nested defs) can + # reference - dropping the declaration while it's still used elsewhere would leave a + # dangling name. Only a genuinely dead type param (never referenced anywhere) is safe + # to remove; err on the side of keeping it otherwise. + return any(isinstance(n, ast.Name) and n.id == name for stmt in body for n in ast.walk(stmt)) + + class RemoveGenerics(SuiteTransformer): """ - Remove bare (non-parametrized) `Generic` base classes. + Remove bare (non-parametrized) `Generic` base classes, and unused PEP 695 + `class Foo[T]:` type params. `Generic[T]` is left alone - the subscript form has real runtime behavior - (`__class_getitem__`) that a bare `Generic` base doesn't add. + (`__class_getitem__`) that a bare `Generic` base doesn't add. Type params are only + dropped when the class isn't exported and isn't subscripted anywhere in this module - + `Foo[int]` relies on `__class_getitem__`, which type params are what provide. """ FLAGS = TransformerFlag.REQUIRES_IMPORT_RESOLVE @@ -28,4 +43,13 @@ def is_enabled(cls, config: TransformConfig, /) -> bool: def visit_ClassDef(self, node: ast.ClassDef): node: ast.ClassDef = super().visit_ClassDef(node) node.bases = [b for b in node.bases if not _is_bare_generic(b)] + + if getattr(node, 'type_params', None): + binding = ref(node).binding + if ( + not binding.exported and not _is_subscripted_anywhere(binding) + and not any(_type_param_used_in_body(tp.name, node.body) for tp in node.type_params) + ): + node.type_params = [] + return node diff --git a/src/terser/_pipeline/transforms/remove_typing_decorators.py b/src/terser/_pipeline/transforms/remove_typing_decorators.py index 2e0cc29..dcbec08 100644 --- a/src/terser/_pipeline/transforms/remove_typing_decorators.py +++ b/src/terser/_pipeline/transforms/remove_typing_decorators.py @@ -1,6 +1,6 @@ from typing import override -from terser.ast import ast +from terser.ast import ast, ref from terser.config import TransformConfig from terser.utils.imports import qualified_name from ._suite import SuiteTransformer, TransformerFlag @@ -8,8 +8,30 @@ _REMOVABLE_NAMES = ("typing.override", "typing_extensions.override", "typing.final", "typing_extensions.final") +def _unreference(decorator: ast.expr): + # Drop the binding reference this decorator was holding (the root `Name` of a bare + # `override` or a dotted `typing.override`), so unused-import cleanup can see it's + # actually unused now that the decorator using it is gone. + node = decorator + while isinstance(node, ast.Attribute): + node = node.value + + if isinstance(node, ast.Name): + try: + ref(node).binding.remove_reference(node) + except AttributeError: + pass + + def _strip(decorator_list: list[ast.expr]) -> list[ast.expr]: - return [d for d in decorator_list if qualified_name(d) not in _REMOVABLE_NAMES] + kept = [] + for d in decorator_list: + if qualified_name(d) in _REMOVABLE_NAMES: + _unreference(d) + else: + kept.append(d) + + return kept class RemoveTypingDecorators(SuiteTransformer): diff --git a/src/terser/ast/ref/_module/_spec.py b/src/terser/ast/ref/_module/_spec.py index 89f16eb..16eb52a 100644 --- a/src/terser/ast/ref/_module/_spec.py +++ b/src/terser/ast/ref/_module/_spec.py @@ -44,7 +44,13 @@ def path(self): @override def resolve(self, module: str): - raise TypeError("Linking is not supported for single module") + # Cross-module linking isn't supported for a single module, but resolving an + # import's own dotted path (for qualified_name-based checks) doesn't need it - + # only reject imports that climb above this (nonexistent) module's root. + if module.startswith(".."): + raise ImportError(f"Could not resolve module: {module}") + + return module[1:] if module.startswith(".") else module @final diff --git a/src/terser/cli/_argparse.py b/src/terser/cli/_argparse.py index cbef585..f786294 100644 --- a/src/terser/cli/_argparse.py +++ b/src/terser/cli/_argparse.py @@ -29,6 +29,17 @@ def __hash__(self) -> int: LiteralGenericAlias = getattr(typing, "_LiteralGenericAlias") +def _parse_bool(value: str) -> bool: + # `type=bool` is a classic argparse footgun: `bool("False")` is `True` (any + # non-empty string is truthy), so `--flag False` would silently become `True`. + if value.lower() in ("true", "1"): + return True + if value.lower() in ("false", "0"): + return False + + raise argparse.ArgumentTypeError(f"invalid boolean value: {value!r}") + + class _ModelArgumentBuilder: __LOCK = object() @@ -89,11 +100,21 @@ def __add_arg(self, parser: ArgParse, field: str, field_info: FieldInfo, model: choices = [True, False] elif isinstance(model, LiteralGenericAlias): choices = get_args(model) + # `model` itself (the `Literal[...]` alias) isn't callable as a `type=` + # converter - convert to whatever type the literal's own values are instead. + model = type(choices[0]) elif isinstance(model, EnumType): choices = list(model.__members__) if not len(choices): choices = None + if isinstance(model, UnionType): + # Resolve `X | None` (e.g. `tuple[int, ...] | None`) to `X` before checking + # if it's a collection type below - `get_origin` on the union itself never + # matches `list`/`set`/`tuple`, so this has to happen first. + non_none = [t for t in get_args(model) if t is not type(None)] + model = non_none[0] if len(non_none) == 1 else None + action, nargs = "store", None if get_origin(model) in (list, set, frozenset, tuple): # 'extend' (not 'append') so repeated uses of the flag accumulate into a @@ -105,17 +126,13 @@ def __add_arg(self, parser: ArgParse, field: str, field_info: FieldInfo, model: elem_types = get_args(model) model = elem_types[0] if elem_types else str - if isinstance(model, UnionType): - non_none = [t for t in get_args(model) if t is not type(None)] - model = non_none[0] if len(non_none) == 1 else None - default = [] if action == "extend" else field_info.get_default(call_default_factory=True) parser.add_argument( "--" + field.replace('_', '-'), action=action, nargs=nargs, default=default, - type=model, # type: ignore[invalid-type] + type=_parse_bool if model is bool else model, # type: ignore[invalid-type] choices=choices, required=field_info.is_required(), help=field_info.description, diff --git a/src/terser/cli/_argv.py b/src/terser/cli/_argv.py index 08b7c52..f6ec234 100644 --- a/src/terser/cli/_argv.py +++ b/src/terser/cli/_argv.py @@ -3,7 +3,7 @@ from alpha93.commons.pydantic import dataclasses from pydantic import BaseModel, ConfigDict, Field -from terser.config import TransformConfig, RemoveAnnotationOptions +from terser.config import TransformConfig, RemoveAnnotationOptions, RemoveDocstringOptions from ._argparse import MutuallyExclusive if TYPE_CHECKING: @@ -109,8 +109,19 @@ def from_argparse(cls, namespace: Namespace, /) -> TerserParsedArguments: remove_argument_annotations=namespace.remove_argument_annotations, remove_attribute_annotations=namespace.remove_attribute_annotations, ) + remove_docstrings = RemoveDocstringOptions( + also_modules=namespace.also_modules, + ) transform_options = TransformConfig( + passes=namespace.passes, optimize=namespace.optimize, + hint_modules=namespace.hint_modules, + # `--target-version` accumulates via the generic list-flag machinery, which + # defaults to `[]` - map that back to `None` (folding disabled) since an empty + # tuple isn't a meaningful version to fold `sys.version_info` comparisons against. + target_version=tuple(namespace.target_version) if namespace.target_version else None, + contracts=namespace.contracts, + apply_contracts=namespace.apply_contracts, remove_literal_statements=namespace.remove_literal_statements, combine_imports=namespace.combine_imports, remove_annotations=remove_annotations if namespace.remove_annotations else False, @@ -118,9 +129,28 @@ def from_argparse(cls, namespace: Namespace, /) -> TerserParsedArguments: remove_explicit_return_none=namespace.remove_explicit_return_none, fold_constants=namespace.fold_constants, remove_debug=namespace.remove_debug, + remove_asserts=namespace.remove_asserts, convert_pass=namespace.convert_pass, + unfold_iife_lambdas=namespace.unfold_iife_lambdas, + remove_type_statements=namespace.remove_type_statements, + convert_early_exits=namespace.convert_early_exits, + convert_to_inline=namespace.convert_to_inline, + convert_to_lambda=namespace.convert_to_lambda, + remove_dummy_assignments=namespace.remove_dummy_assignments, + remove_docstrings=remove_docstrings if namespace.remove_docstrings else False, + respect_all=namespace.respect_all, + cleanup_local_imports=namespace.cleanup_local_imports, + remove_typing_decorators=namespace.remove_typing_decorators, + remove_overloads=namespace.remove_overloads, + remove_generics=namespace.remove_generics, + remove_typing_classes=namespace.remove_typing_classes, + convert_typing_constructors=namespace.convert_typing_constructors, + convert_typing_extensions=namespace.convert_typing_extensions, + convert_dynamic_attribute_access=namespace.convert_dynamic_attribute_access, remove_empty_exc_brackets=namespace.remove_empty_exc_brackets, convert_posargs=namespace.convert_posargs, + remove_dunder_all=namespace.remove_dunder_all, + remove_dunder_all_modules=namespace.remove_dunder_all_modules, ) mangling_options = ManglingOptions( diff --git a/src/terser/config.py b/src/terser/config.py index 073659d..f27c12f 100644 --- a/src/terser/config.py +++ b/src/terser/config.py @@ -32,6 +32,17 @@ class TransformConfig: passes: int = 5 optimize: Literal[-1, 0, 1, 2] = -1 + hint_modules: list[str] = field(default_factory=list) + """Extra dotted module paths whose members are recognized as `terser_hints` decorators + (e.g. `preserve_docstring`, `preserve_annotations`, `constant`), in addition to the + built-in `terser_hints` package""" + + target_version: tuple[int, ...] | None = None + """Target Python version (e.g. `(3, 12)`) the minified output will run under - folds + `sys.version_info (...)` comparisons against a literal tuple to a constant bool. + `None` (the default) leaves these comparisons untouched, since the runtime version + isn't known.""" + contracts: list[str] = field(default_factory=lambda: [ "typing.cast(_, value) -> value", "typing.assert_never(_) -> None", @@ -128,7 +139,16 @@ class TransformConfig: """Convert positional-only arguments to normal arguments""" remove_dunder_all: bool = False - """Remove the top-level `__all__` assignment. Unsafe across module boundaries: another - module doing `from this_module import *` relies on `__all__` (falling back to "no names" - when every top-level name is prefixed with an underscore), which a per-module pass run - before project-wide linking has no way to see.""" + """Remove the top-level `__all__` assignment everywhere. Unsafe across module boundaries: + another module doing `from this_module import *` relies on `__all__` (falling back to "no + names" when every top-level name is prefixed with an underscore), which a per-module pass + run before project-wide linking has no way to see. See `remove_dunder_all_modules` for a + safer, per-module opt-in instead of this project-wide switch.""" + + remove_dunder_all_modules: list[str] = field(default_factory=list) + """Glob patterns (matched against each module's dotted path) whitelisting specific modules + where `__all__` removal is safe (e.g. an app entry point that's never `from x import *`-ed + elsewhere), without enabling `remove_dunder_all` project-wide. Opt-in per module, unlike + `preserve_locals`/`preserve_globals` (which are opt-out from a rename-everything default), + since `__all__` can be relied on internally within a module too (e.g. `__all__.append(...)`, + already guarded against separately - see `RemoveAll`).""" diff --git a/src/terser/utils/hints.py b/src/terser/utils/hints.py new file mode 100644 index 0000000..5ef56bb --- /dev/null +++ b/src/terser/utils/hints.py @@ -0,0 +1,13 @@ +from terser.ast import ast +from terser.config import TransformConfig +from .imports import qualified_name + + +def is_hinted(decorator_list: list[ast.expr], hint: str, config: TransformConfig) -> bool: + """ + Check if any decorator in `decorator_list` matches the terser hint named `hint` + (`terser_hints.`, or `.` for any alias in `config.hint_modules`). + """ + names = {f"terser_hints.{hint}"} + names.update(f"{module}.{hint}" for module in config.hint_modules) + return any(qualified_name(d) in names for d in decorator_list) diff --git a/src/terser/utils/imports.py b/src/terser/utils/imports.py index aa024ea..24c4514 100644 --- a/src/terser/utils/imports.py +++ b/src/terser/utils/imports.py @@ -1,5 +1,6 @@ from terser.ast import ast, ref from .._pipeline.resolver.binding import ImportBinding +from .._pipeline.resolver.dynamic_import import match_dynamic_import_call def qualified_name(node: ast.expr, /) -> str | None: @@ -7,7 +8,10 @@ def qualified_name(node: ast.expr, /) -> str | None: Resolve a `Name` or dotted `Attribute` expression to the dotted path of the import it refers to (e.g. `cast` after `from typing import cast` or `typing.cast` both resolve to `"typing.cast"`), using the already-computed - name bindings. Returns None if the root name isn't an import, or its + name bindings. Also recognizes an inline dynamic-import call directly (e.g. + `__import__("typing").TYPE_CHECKING`), with no binding involved. + + Returns None if the root name isn't an import (or dynamic-import call), or its binding has no name. """ @@ -17,6 +21,9 @@ def qualified_name(node: ast.expr, /) -> str | None: node = node.value attrs.reverse() + if (source_module := match_dynamic_import_call(node)) is not None: + return f"{source_module}.{'.'.join(attrs)}" if attrs else source_module + if not isinstance(node, ast.Name): return None @@ -37,8 +44,9 @@ def qualified_name(node: ast.expr, /) -> str | None: # `attrs` non-empty means this was a dotted access off an `import x` binding # (e.g. `typing.cast`) - the module identity comes from source_module, not - # the (possibly aliased) local name. With no attrs, this is a bare `Name` - # from a `from x import y` binding, where binding.name is the local - # (possibly aliased) symbol name. - tail = ".".join(attrs) if attrs else binding.name + # the (possibly aliased) local name. With no attrs, this is a bare `Name` - use + # `remote_name` (the name as written in the source module) when the binding has + # one (e.g. `override` for `from typing import override as ov`), since `.name` may + # be a local alias or a later-mangled name that means nothing in the source module. + tail = ".".join(attrs) if attrs else (binding.remote_name or binding.name) return f"{source_module}.{tail}" diff --git a/src/terser_hints/__init__.py b/src/terser_hints/__init__.py index 5472b64..4c8c203 100644 --- a/src/terser_hints/__init__.py +++ b/src/terser_hints/__init__.py @@ -12,4 +12,23 @@ def preserve_docstring(obj: _T, /) -> _T: return obj -__all__ = ("preserve_docstring",) +def preserve_annotations(obj: _T, /) -> _T: + """ + Marker decorator: tells terser to keep this function/class's type annotations + even when annotation removal is otherwise enabled. Detected statically, + a no-op at runtime. + """ + return obj + + +def constant(fn): + """ + Marker decorator: tells terser this function is only ever called once, to + compute a constant - immediately call it and rebind its name to the result, + the same as the `@lambda _: _()` idiom. Detected statically; at runtime this + just calls `fn` once and returns its result. + """ + return fn() + + +__all__ = ("preserve_docstring", "preserve_annotations", "constant")