Codeseys/composer-replication-framework

1"""Tests for the layered HintGenerator architecture (ADR-009).2 3Covers ADR-009 acceptance gates:4  - gate 1: HintGenerator Protocol; layers satisfy it (runtime_checkable).5  - gate 2: TemplateHintGenerator is byte-identical to the existing dispatch()6    for all 5 registered kinds (no regression).7  - gate 3: CompositeHintGenerator tries layers cost-first — a tool_not_found8    site is served by the template layer (no LLM call); a style site falls9    through to the judge layer.10  - gate 4: LLMJudgeHintGenerator caches (second identical call = zero11    completions).12  - gate 5: as_collator_hook() matches CollatorConfig.hint_generator's13    (error_kind, error_meta) -> str | None signature.14 15All CPU-only, no network (LLM layer is a stub).16"""17from __future__ import annotations18 19from composer_replication.hint_generator import (20    HINT_TEMPLATES,21    CompositeHintGenerator,22    HintGenerator,23    LLMJudgeHintGenerator,24    RawErrorHintGenerator,25    TemplateHintGenerator,26    default_composite,27    dispatch,28)29 30 31# --- gate 1: Protocol -------------------------------------------------------32 33def test_layers_satisfy_protocol():34    assert isinstance(TemplateHintGenerator(), HintGenerator)35    assert isinstance(RawErrorHintGenerator(), HintGenerator)36    assert isinstance(LLMJudgeHintGenerator(), HintGenerator)37    assert isinstance(CompositeHintGenerator([]), HintGenerator)38 39 40# --- gate 2: template byte-identity ----------------------------------------41 42def test_template_layer_byte_identical_to_dispatch():43    tmpl = TemplateHintGenerator()44    meta = {45        "available_tools": ["read", "write"],46        "tool_name": "frobnicate",47        "tool_schema": {"x": "int"},48        "error_message": "boom",49    }50    for kind in HINT_TEMPLATES:51        ctx = dict(meta)52        ctx.setdefault("error_kind", kind)53        expected = dispatch(kind, ctx)54        got = tmpl.generate(kind, meta)55        assert got == expected, f"template layer drifted from dispatch for {kind}"56 57 58def test_template_layer_returns_none_for_unknown_kind():59    assert TemplateHintGenerator().generate("totally_unknown_kind", {}) is None60 61 62# --- gate 3: cost-ordered composite ----------------------------------------63 64def test_composite_serves_tool_error_from_template_no_llm():65    calls = {"n": 0}66 67    def fake_complete(prompt: str) -> str:68        calls["n"] += 169        return "LLM HINT"70 71    comp = default_composite(llm_complete=fake_complete)72    hint = comp.generate("tool_not_found", {"available_tools": ["read", "write"]})73    assert hint is not None74    assert "Available tools" in hint  # template output75    assert calls["n"] == 0, "LLM judge must NOT be called for a template-covered site"76 77 78def test_composite_falls_through_to_judge_for_uncovered_site():79    calls = {"n": 0}80 81    def fake_complete(prompt: str) -> str:82        calls["n"] += 183        return "Be more concise; you repeated the same explanation."84 85    comp = default_composite(llm_complete=fake_complete, enable_raw_error=False)86    # 'verbose_communication' has no template and no error_message -> judge.87    hint = comp.generate("verbose_communication", {})88    assert hint == "Be more concise; you repeated the same explanation."89    assert calls["n"] == 190 91 92def test_raw_error_layer_covers_unmatched_site_with_message():93    comp = default_composite()  # no LLM94    hint = comp.generate("weird_unmapped_error", {"error_message": "Segfault at 0x0"})95    assert hint is not None96    assert "Segfault at 0x0" in hint97 98 99def test_composite_returns_none_when_all_layers_defer():100    comp = default_composite()  # templates + raw-error, no LLM101    # unknown kind + no message -> nothing fires102    assert comp.generate("unknown", {}) is None103 104 105# --- gate 4: LLM-judge cache ------------------------------------------------106 107def test_llm_judge_caches_in_memory(tmp_path):108    calls = {"n": 0}109 110    def fake_complete(prompt: str) -> str:111        calls["n"] += 1112        return f"hint #{calls['n']}"113 114    judge = LLMJudgeHintGenerator(fake_complete, cache_dir=str(tmp_path))115    meta = {"error_message": "X"}116    h1 = judge.generate("k", meta)117    h2 = judge.generate("k", meta)  # identical -> cache hit118    assert h1 == h2119    assert calls["n"] == 1, "second identical call must hit cache (zero completions)"120 121 122def test_llm_judge_disk_cache_survives_new_instance(tmp_path):123    calls = {"n": 0}124 125    def fake_complete(prompt: str) -> str:126        calls["n"] += 1127        return "persisted hint"128 129    j1 = LLMJudgeHintGenerator(fake_complete, cache_dir=str(tmp_path))130    j1.generate("k", {"error_message": "X"})131    # fresh instance, same cache dir -> disk hit, no completion132    j2 = LLMJudgeHintGenerator(fake_complete, cache_dir=str(tmp_path))133    h = j2.generate("k", {"error_message": "X"})134    assert h == "persisted hint"135    assert calls["n"] == 1136 137 138def test_llm_judge_disabled_when_no_complete():139    assert LLMJudgeHintGenerator(None).generate("k", {"error_message": "X"}) is None140 141 142# --- gate 5: collator-hook signature ---------------------------------------143 144def test_as_collator_hook_matches_collator_signature():145    comp = default_composite()146    hook = comp.as_collator_hook()147    # CollatorConfig.hint_generator is Callable[[str, dict], str | None]148    out = hook("tool_not_found", {"available_tools": ["read"]})149    assert isinstance(out, str)150    out_none = hook("unknown", {})151    assert out_none is None152 153 154def test_as_collator_hook_drops_into_collator_config():155    """The hook is accepted by CollatorConfig without changes."""156    from composer_replication.trainer.data_collator import CollatorConfig157 158    comp = default_composite()159    cfg = CollatorConfig(hint_generator=comp.as_collator_hook())160    assert cfg.hint_generator is not None161    assert cfg.hint_generator("json_decode", {}) is not None  # template fires162