lablab-ai-amd-developer-hackathon/gpu-goblin

1"""Tests for the defensive misnested-arg extraction in benchmark + profile_run.2 3Live AMD-GPU lesson: Qwen2.5-7B (and probably others) occasionally JSON-nests4``steps`` / ``cache`` *inside* the ``config`` dict instead of at the top level5alongside it. WorkloadConfig strict-validates extras, so without this defense6the call errors out and a tool slot is wasted. The well-tuned scenario run7on 2026-05-07 burned two of the eight available slots on this exact mistake;8fixing it costs nothing and saves the audit.9"""10 11from __future__ import annotations12 13import shutil14 15from agent.tools import call16 17 18def _baseline_config() -> dict:19    return {20        "model_name": "Qwen/Qwen2.5-7B-Instruct",21        "batch_size": 4,22        "precision": "fp16",23        "attention_impl": "eager",24        "dataloader_workers": 0,25    }26 27 28class TestBenchmarkMisnestedArgs:29    def setup_method(self) -> None:30        # Each test starts with an empty cache so cache-hit doesn't mask the31        # behavior under test.32        shutil.rmtree("bench_cache", ignore_errors=True)33 34    def test_steps_nested_in_config_is_extracted(self) -> None:35        """Old behavior: ``WorkloadConfig`` validation explodes with36        'Extra inputs are not permitted [steps]'. New behavior: defensive37        extraction pulls ``steps`` back to the top-level arg, call succeeds.38        """39        cfg = {**_baseline_config(), "steps": 25}40        result = call("benchmark", config=cfg)41        assert result.ok, result.error42        assert result.result["steps"] == 2543 44    def test_cache_nested_in_config_is_extracted(self) -> None:45        cfg = {**_baseline_config(), "cache": False}46        result = call("benchmark", config=cfg)47        assert result.ok, result.error48 49    def test_force_rerun_nested_in_config_is_extracted(self) -> None:50        cfg = {**_baseline_config(), "force_rerun": True}51        result = call("benchmark", config=cfg)52        assert result.ok, result.error53 54    def test_explicit_top_level_wins_over_nested(self) -> None:55        """If caller passes BOTH (config has steps + top-level steps), the56        explicit non-default top-level wins. Defensive code is for the57        accident case, not for letting nesting silently override."""58        cfg = {**_baseline_config(), "steps": 25}59        result = call("benchmark", config=cfg, steps=37)60        assert result.ok, result.error61        assert result.result["steps"] == 3762 63    def test_all_three_nested_at_once(self) -> None:64        """The exact failure mode from the live run: model nested three65        runtime args inside config. All three should get pulled out.66        """67        cfg = {68            **_baseline_config(),69            "steps": 30,70            "cache": False,71            "force_rerun": True,72        }73        result = call("benchmark", config=cfg)74        assert result.ok, result.error75        assert result.result["steps"] == 3076 77 78class TestProfileRunMisnestedArgs:79    def test_steps_nested_in_config_is_extracted(self) -> None:80        cfg = {**_baseline_config(), "steps": 7}81        result = call("profile_run", config=cfg)82        assert result.ok, result.error83        assert result.result["steps"] == 784 85    def test_explicit_top_level_wins(self) -> None:86        cfg = {**_baseline_config(), "steps": 7}87        result = call("profile_run", config=cfg, steps=15)88        assert result.ok, result.error89        assert result.result["steps"] == 1590 91    def test_clean_config_unaffected(self) -> None:92        """Sanity: when nothing is misnested, behavior is unchanged."""93        result = call("profile_run", config=_baseline_config())94        assert result.ok, result.error95        assert result.result["steps"] == 10  # default96