alvinwan · alvinwan · Apr 9, 2026 · Apr 9, 2026 · Apr 9, 2026 · Apr 9, 2026
diff --git a/benchmarks/README.md b/benchmarks/README.md
@@ -97,44 +97,36 @@ Wheel-specific failures:
 
 | Input | pymini | pyminifier | python-minifier |
 | --- | ---: | ---: | ---: |
-| pyminifier.py | 3.0 ms | 0.8 ms | 2.6 ms |
-| pyminify.py | 7.9 ms | 2.0 ms | 6.7 ms |
-| click | 855.2 ms | failed | 823.9 ms |
-| pytest | 3.258 s | failed | 3.156 s |
-| TexSoup | 218.6 ms | failed | 251.1 ms |
-| timefhuman | 441.5 ms | failed | 450.6 ms |
-| pyminifier | 196.7 ms | 68.5 ms | 191.7 ms |
-| rich | 3.226 s | failed | 3.080 s |
+| pyminifier.py | 9.4 ms | 1.3 ms | 5.5 ms |
+| pyminify.py | 17.2 ms | 4.2 ms | 16.2 ms |
+| click | 4.943 s | failed | 6.478 s |
+| pytest | 32.292 s | failed | 23.964 s |
+| pyminifier | 1.256 s | 131.1 ms | 610.8 ms |
 
 pyminifier minification fails on `__init__.py` with
   `TypeError: 'NoneType' object is not subscriptable`.
 
-The single-file rows come from [benchmark_speed.py](./benchmark_speed.py). The
-package rows are averages of three fresh-process runs from the same
-environment used for the compression comparison. `pymini` used package mode
-with `--rename-modules --rename-global-variables --rename-arguments`, while
-the baseline tools minified each file independently in the preserved package
-tree. The `click`, `pytest`, and `pyminifier` rows use the checked-in fixtures
-under `.bench-repos`; the other package rows use local package checkouts.
+The rows above come from [benchmark_speed.py](./benchmark_speed.py). The
+single-file rows average ten in-process runs after one warmup. The checked-in
+package rows average three in-process runs after one warmup over the fixtures
+under `.bench-repos`. `pymini` uses package mode with
+`--rename-modules --rename-global-variables --rename-arguments`, while the
+baseline tools minify each file independently in the preserved package tree.
 
 # Reproduce
 
 Recompute the speed measurements with:
 
 ```bash
 python3 -m pip install -e ".[dev]" python-minifier
-git clone https://github.com/liftoff/pyminifier /tmp/pyminifier
-PYTHONPATH=. .venv/bin/python benchmarks/benchmark_speed.py --pyminifier-root /tmp/pyminifier
+PYTHONPATH=. .venv/bin/python benchmarks/benchmark_speed.py
 ```
 
-The larger package comparisons in this file were run against these checkouts:
+The checked-in speed fixtures in this file live under `.bench-repos`:
 
-- `TexSoup`
-- `timefhuman`
-- `pyminifier`
-- `rich`
 - `click`
 - `pytest`
+- `pyminifier`
 
 # Validation
 

diff --git a/benchmarks/benchmark_speed.py b/benchmarks/benchmark_speed.py
@@ -2,10 +2,13 @@
 from __future__ import annotations
 
 import argparse
+import collections
+import collections.abc
 import importlib
 import shutil
 import sys
 import tempfile
+import warnings
 from pathlib import Path
 from statistics import mean
 from time import perf_counter
@@ -18,13 +21,26 @@
 ROOT = Path(__file__).resolve().parents[1]
 EXAMPLE_DIR = ROOT / "tests" / "examples"
 DEFAULT_TEXSOUP_ROOT = Path("/tmp/pymini-texsoup-repo/TexSoup")
-DEFAULT_PYMINIFIER_ROOT = Path("/tmp/pymini-pyminifier-src/pyminifier-2.1")
+DEFAULT_PYMINIFIER_ROOT = (
+    ROOT / ".bench-repos" / "pyminifier-tool"
+    if (ROOT / ".bench-repos" / "pyminifier-tool").exists()
+    else Path("/tmp/pymini-pyminifier-src/pyminifier-2.1")
+)
+DEFAULT_FIXTURE_ROOT = ROOT / ".bench-repos"
+EXAMPLE_BENCHMARKS = ("pyminifier.py", "pyminify.py")
+FIXTURE_BENCHMARKS = (
+    ("click", "click"),
+    ("pytest", "pytest"),
+    ("pyminifier", "pyminifier-tool/pyminifier"),
+)
 PYMINI_AGGRESSIVE_OPTIONS = {
     "keep_module_names": False,
     "keep_global_variables": False,
     "rename_arguments": True,
 }
 
+warnings.filterwarnings("ignore", category=SyntaxWarning)
+
 
 def benchmark_transform(
     transform,
@@ -78,6 +94,8 @@ def factory(path: Path):
 def load_pyminifier(pyminifier_root: Path):
     if not pyminifier_root.exists():
         return None
+    if not hasattr(collections, "Iterable"):
+        collections.Iterable = collections.abc.Iterable
     sys.path.insert(0, str(pyminifier_root))
     try:
         minification = importlib.import_module("pyminifier.minification")
@@ -96,6 +114,44 @@ def transform(source: str) -> str:
     return factory
 
 
+def benchmark_package_filewise(
+    package_root: Path,
+    *,
+    factory_loader,
+    iterations: int,
+    warmup: int,
+) -> dict[str, float]:
+    paths = [Path(path) for path in resolve_python_files(str(package_root))[0]]
+    transforms = []
+    raw_bytes = 0
+    factory = factory_loader()
+    for path in paths:
+        source = path.read_text(encoding="utf-8")
+        raw_bytes += len(source.encode())
+        transforms.append((factory(path), source))
+
+    for _ in range(warmup):
+        for transform, source in transforms:
+            transform(source)
+
+    samples = []
+    output_bytes = 0
+    for _ in range(iterations):
+        start = perf_counter()
+        outputs = [transform(source) for transform, source in transforms]
+        samples.append(perf_counter() - start)
+        output_bytes = sum(len(output.encode()) for output in outputs)
+
+    avg = mean(samples)
+    return {
+        "files": float(len(paths)),
+        "bytes": float(raw_bytes),
+        "output_bytes": float(output_bytes),
+        "avg_ms": avg * 1000,
+        "throughput_kb_s": (raw_bytes / 1024) / avg,
+    }
+
+
 def benchmark_package_api(
     package_root: Path,
     *,
@@ -176,7 +232,10 @@ def print_example_results(
 
     print("Single-file API benchmarks")
     print("input\ttool\tinput_bytes\toutput_bytes\tavg_ms\tthroughput_kb_s")
-    for path in sorted(EXAMPLE_DIR.glob("*.py")):
+    for name in EXAMPLE_BENCHMARKS:
+        path = EXAMPLE_DIR / name
+        if not path.exists():
+            continue
         source = path.read_text(encoding="utf-8")
         for tool_name, factory in tool_factories:
             result = benchmark_transform(
@@ -195,6 +254,85 @@ def print_example_results(
             )
 
 
+def print_fixture_results(
+    fixture_root: Path,
+    *,
+    package_iterations: int,
+    warmup: int,
+    pyminifier_root: Path,
+) -> None:
+    if not fixture_root.exists():
+        print(f"Checked-in fixture benchmarks skipped: {fixture_root} does not exist")
+        return
+
+    python_minifier = load_python_minifier()
+    pyminifier_factory = load_pyminifier(pyminifier_root)
+
+    print()
+    print("Checked-in fixture package benchmarks")
+    print("name\ttool\tfiles\tinput_bytes\toutput_bytes\tavg_ms\tthroughput_kb_s\tstatus")
+    for name, relative_path in FIXTURE_BENCHMARKS:
+        package_root = fixture_root / relative_path
+        if not package_root.exists():
+            continue
+
+        pymini_result = benchmark_package_api(
+            package_root,
+            iterations=package_iterations,
+            warmup=warmup,
+        )
+        print(
+            f"{name}\tpymini\t"
+            f"{int(pymini_result['files'])}\t"
+            f"{int(pymini_result['bytes'])}\t"
+            f"{int(pymini_result['output_bytes'])}\t"
+            f"{pymini_result['avg_ms']:.3f}\t"
+            f"{pymini_result['throughput_kb_s']:.1f}\tok"
+        )
+
+        if python_minifier is not None:
+            result = benchmark_package_filewise(
+                package_root,
+                factory_loader=load_python_minifier,
+                iterations=package_iterations,
+                warmup=warmup,
+            )
+            print(
+                f"{name}\tpython-minifier\t"
+                f"{int(result['files'])}\t"
+                f"{int(result['bytes'])}\t"
+                f"{int(result['output_bytes'])}\t"
+                f"{result['avg_ms']:.3f}\t"
+                f"{result['throughput_kb_s']:.1f}\tok"
+            )
+
+        if pyminifier_factory is None:
+            continue
+
+        try:
+            result = benchmark_package_filewise(
+                package_root,
+                factory_loader=lambda: pyminifier_factory,
+                iterations=package_iterations,
+                warmup=warmup,
+            )
+        except Exception as exc:
+            print(
+                f"{name}\tpyminifier\t-\t-\t-\t-\t-\t"
+                f"failed: {type(exc).__name__}: {exc}"
+            )
+            continue
+
+        print(
+            f"{name}\tpyminifier\t"
+            f"{int(result['files'])}\t"
+            f"{int(result['bytes'])}\t"
+            f"{int(result['output_bytes'])}\t"
+            f"{result['avg_ms']:.3f}\t"
+            f"{result['throughput_kb_s']:.1f}\tok"
+        )
+
+
 def print_package_results(
     texsoup_root: Path,
     *,
@@ -247,6 +385,12 @@ def main() -> int:
         default=DEFAULT_PYMINIFIER_ROOT,
         help="Path to a pyminifier source checkout for baseline single-file benchmarks.",
     )
+    parser.add_argument(
+        "--fixture-root",
+        type=Path,
+        default=DEFAULT_FIXTURE_ROOT,
+        help="Path to the checked-in fixture package checkouts used for package speed rows.",
+    )
     parser.add_argument(
         "--example-iterations",
         type=int,
@@ -284,6 +428,12 @@ def main() -> int:
         package_cli_iterations=args.package_cli_iterations,
         warmup=args.warmup,
     )
+    print_fixture_results(
+        args.fixture_root,
+        package_iterations=args.package_api_iterations,
+        warmup=args.warmup,
+        pyminifier_root=args.pyminifier_root,
+    )
     return 0