pennaburry/parallel-constrained-decoding

1"""2Benchmark runner comparing Autoregressive Generation vs.3Parallel Constrained Decision Engine on Apple Silicon.4"""5 6import time7import json8import argparse9from typing import Dict, Any, List10from core.schema import StructuredSchema11from core.engine import run_naive_generation, run_parallel_generation, get_engine12 13 14def compare_single(context: str, schema_dict: Dict[str, Any]) -> Dict[str, Any]:15    """Runs both engines on the exact same problem prompt and returns side-by-side metrics."""16    schema = StructuredSchema(schema_dict)17    18    # 1. Run Autoregressive Baseline19    naive_res = run_naive_generation(context, schema)20    21    # 2. Run Parallel Constrained Engine22    parallel_res = run_parallel_generation(context, schema)23    24    speedup = naive_res["elapsed_ms"] / max(parallel_res["elapsed_ms"], 1.0)25    steps_speedup = naive_res["sequential_forward_passes"] / max(parallel_res["sequential_forward_passes"], 1.0)26    27    return {28        "speedup_multiplier": round(speedup, 1),29        "steps_reduction": round(steps_speedup, 1),30        "naive": naive_res,31        "parallel": parallel_res,32        # Backward compatibility33        "rlcd": parallel_res34    }35 36 37def run_benchmark_suite(preset_paths: List[str], warmup: bool = True) -> List[Dict[str, Any]]:38    print("=" * 70)39    print("Parallel Constrained vs. Autoregressive Generation Benchmark")40    print("=" * 70)41    42    get_engine()43    44    if warmup:45        print("\n[+] Warming up GPU compute graphs...")46        with open(preset_paths[0]) as f:47            p = json.load(f)48        compare_single(p["context"], p["schema"])49        print("[+] Warmup complete.\n")50        51    results = []52    for path in preset_paths:53        with open(path) as f:54            preset = json.load(f)55            56        print(f"--> Running preset: {preset['title']} ({len(preset['schema'])} fields)...")57        comp = compare_single(preset["context"], preset["schema"])58        comp["preset_id"] = preset["id"]59        comp["preset_title"] = preset["title"]60        results.append(comp)61        62        n = comp["naive"]63        r = comp["parallel"]64        print(f"    Autoregressive Baseline : {n['elapsed_ms']:>8.1f} ms | {n['total_tokens']:>3} tokens ({n['tokens_per_second']} tok/s) | Passes: {n['sequential_forward_passes']}")65        print(f"    Parallel Constrained    : {r['elapsed_ms']:>8.1f} ms |   0 tokens (O(1))           | Passes: {r['sequential_forward_passes']}")66        print(f"    >> SPEEDUP: {comp['speedup_multiplier']}x faster (Step reduction: {comp['steps_reduction']}x)")67        print(f"    >> Schema match: Naive={n['schema_match']} | Parallel={r['schema_match']} (100% guaranteed)")68        print("-" * 70)69        70    return results71 72 73if __name__ == "__main__":74    parser = argparse.ArgumentParser(description="Run Parallel vs Autoregressive LLM JSON benchmark")75    parser.add_argument("--presets", nargs="+", default=[76        "presets/fintech_fraud.json",77        "presets/support_triage.json",78        "presets/high_cardinality_255.json"79    ])80    args = parser.parse_args()81    run_benchmark_suite(args.presets)82