PerturbReason/PerturbReason_dataset_code
050
1#!/usr/bin/env python32"""3qualitative/case_study_select.py4==================================5Phase 3 — Representative case-study selector.6 7Strategy for selecting representative examples from ~40K samples:8 9 1. For each target category (ANSWER_BUG, noisy path, ambiguity, etc.),10 filter samples matching the criteria.11 2. Compute a "typicality score" = closeness to the category median on12 key numeric features (edge_f1, hallucination_rate, num_model_triplets).13 3. Among the top-K most typical, pick the one with the shortest reasoning14 (for readability in the paper) and that has a moderate pathway length (3-5 edges).15 4. Output the selected samples with full context (GT, prediction, pathway).16 17Usage::18 19 python qualitative/case_study_select.py \\20 --noisy-csv eval_v3/eval_v3_output/qwen_base_noisy_0326_eval/samples_with_rescue_*.csv \\21 --hidden-csv eval_v3/eval_v3_output/qwen_base_hidden_0326_eval/samples_with_rescue_*.csv \\22 --gt-noisy baseline_collections/dataset_0326_noisy_hidden/noisy \\23 --gt-hidden baseline_collections/dataset_0326_noisy_hidden/hidden \\24 --pred-noisy baseline_collections/qwen_base_official/noisy_output_0326 \\25 --pred-hidden baseline_collections/qwen_base_official/hidden_output_0326 \\26 -o qualitative/output/case_studies27"""28from __future__ import annotations29 30import argparse31import json32from collections import defaultdict33from pathlib import Path34from typing import Any, Dict, List, Optional, Tuple35 36import matplotlib37matplotlib.use("Agg")38import matplotlib.pyplot as plt39import matplotlib.patches as mpatches40import numpy as np41 42from utils import (43 ERROR_LABEL_ORDER, PALETTE, SPLIT_ORDER,44 build_eval_sample_key, build_json_sample_key,45 get_row_file_name, get_split_display_name,46 extract_answer_from_text, extract_thinking_from_text, extract_triplets_from_text,47 iter_gt_files, iter_pred_files, load_samples_csv, parse_prompt_edges,48 parse_prompt_metadata, safe_bool, safe_float, safe_int,49 save_figure, set_model_tag, setup_figure_style, tagged_title,50 write_markdown_table,51)52 53 54def parse_args():55 p = argparse.ArgumentParser(description="Case study selector")56 p.add_argument("--noisy-csv", required=True)57 p.add_argument("--hidden-csv", required=True)58 p.add_argument("--gt-noisy", required=True)59 p.add_argument("--gt-hidden", required=True)60 p.add_argument("--pred-noisy", required=True)61 p.add_argument("--pred-hidden", required=True)62 p.add_argument("-o", "--output-dir", default="qualitative/output/case_studies")63 p.add_argument("--n-per-category", type=int, default=3, help="Cases per category")64 p.add_argument("--model-name", default="", help="Model name for figure/table titles")65 return p.parse_args()66 67 68# ══════════════════════════════════════════════════════════════69# Data loading70# ══════════════════════════════════════════════════════════════71 72def build_gt_index(gt_dir: str) -> Dict[Tuple[str, str, str], Dict]:73 """Index GT records by stable (split, pert_type, sample_id)."""74 index = {}75 for split, pert_type, fpath in iter_gt_files(gt_dir):76 with open(fpath, "r") as f:77 for idx, line in enumerate(f):78 line = line.strip()79 if not line:80 continue81 record = json.loads(line)82 index[build_json_sample_key(split, pert_type, record, idx)] = record83 return index84 85 86def build_pred_index(pred_dir: str) -> Dict[Tuple[str, str, str], Dict]:87 """Index prediction records by stable (split, pert_type, sample_id)."""88 index = {}89 for split, pert_type, fpath in iter_pred_files(pred_dir):90 with open(fpath, "r") as f:91 for idx, line in enumerate(f):92 line = line.strip()93 if not line:94 continue95 record = json.loads(line)96 index[build_json_sample_key(split, pert_type, record, idx)] = record97 return index98 99 100def resolve_key(row: Dict[str, str]) -> Tuple[str, str, str]:101 """Get a stable (split, pert_type, sample_id) key from an eval CSV row."""102 return build_eval_sample_key(row)103 104 105# ══════════════════════════════════════════════════════════════106# Typicality scoring107# ══════════════════════════════════════════════════════════════108 109FEATURES = ["edge_f1_strict", "hallucination_rate", "num_model_triplets"]110 111 112def compute_typicality(candidates: List[Dict[str, str]]) -> List[Tuple[float, int]]:113 """114 For a list of eval rows, compute typicality = L2 distance to the feature median.115 Returns list of (distance, index) sorted by distance (most typical first).116 """117 if not candidates:118 return []119 120 feature_vecs = []121 for r in candidates:122 vec = [safe_float(r.get(f), 0) for f in FEATURES]123 feature_vecs.append(vec)124 125 arr = np.array(feature_vecs)126 median = np.median(arr, axis=0)127 # Normalise by range128 ranges = np.ptp(arr, axis=0)129 ranges[ranges == 0] = 1.0130 normalised = (arr - median) / ranges131 distances = np.linalg.norm(normalised, axis=1)132 133 scored = [(distances[i], i) for i in range(len(candidates))]134 scored.sort(key=lambda x: x[0])135 return scored136 137 138def select_representatives(candidates: List[Dict[str, str]], n: int = 3,139 prefer_path_len: Tuple[int, int] = (3, 6)) -> List[Dict[str, str]]:140 """141 Select n representative samples from candidates:142 1. Score by typicality (closeness to median)143 2. Among top-20 most typical, prefer moderate model chain length144 3. Return top n145 """146 if len(candidates) <= n:147 return candidates148 149 scored = compute_typicality(candidates)150 # Take top-20 most typical151 top_k = min(20, len(scored))152 top_indices = [idx for _, idx in scored[:top_k]]153 154 # Among top, prefer moderate chain length155 def chain_preference(row):156 cl = safe_int(row.get("num_model_triplets"), 0)157 lo, hi = prefer_path_len158 if lo <= cl <= hi:159 return 0160 return abs(cl - (lo + hi) / 2)161 162 ranked = sorted(top_indices, key=lambda i: chain_preference(candidates[i]))163 return [candidates[i] for i in ranked[:n]]164 165 166# ══════════════════════════════════════════════════════════════167# Case categories168# ══════════════════════════════════════════════════════════════169 170def define_categories():171 """Define case study categories and their selection criteria."""172 return {173 "answer_bug": {174 "description": "ANSWER_BUG: Model reasons correctly but writes wrong answer label",175 "filter": lambda r: (176 r.get("error_label", "") == "WRONG_CORRECT_CHAIN"177 and r.get("llm_rescue_verdict", "") == "ANSWER_BUG"178 and safe_int(r.get("num_model_triplets")) >= 2179 ),180 },181 "noisy_path": {182 "description": "NOISY PATH: Too many retrieved edges overwhelm the model",183 "filter": lambda r: (184 r.get("error_label", "") in ("WRONG_NO_PATH", "WRONG_HALLUCINATED_EDGE")185 and not safe_bool(r.get("answer_correct"))186 ),187 },188 "ambiguity_alternate": {189 "description": "AMBIGUITY: Model finds valid alternative pathway",190 "filter": lambda r: (191 r.get("llm_rescue_verdict", "") == "ALTERNATE"192 or r.get("error_label", "") == "WRONG_CORRECT_CHAIN_ALTERNATE"193 ),194 },195 "correct_exemplar": {196 "description": "CORRECT: Clean correct prediction with valid chain",197 "filter": lambda r: (198 r.get("error_label", "") == "CORRECT"199 and safe_float(r.get("edge_f1_strict")) >= 0.8200 ),201 },202 "sign_flip": {203 "description": "SIGN FLIP: Wrong answer due to sign error in reasoning",204 "filter": lambda r: (205 r.get("error_label", "") == "WRONG_SIGN_FLIP"206 and safe_int(r.get("sign_flips")) >= 1207 ),208 },209 }210 211 212# ══════════════════════════════════════════════════════════════213# Pathway rendering214# ══════════════════════════════════════════════════════════════215 216def render_pathway_diagram(gt_triplets, model_triplets, prompt_edges,217 pert_gene, effect_gene, title, out_path: Path):218 """219 Render a pathway comparison diagram:220 - GT edges in green221 - Model edges in blue222 - Hallucinated edges in red223 - Prompt edges (context) in light grey224 """225 setup_figure_style()226 try:227 import networkx as nx228 except ImportError:229 print(" networkx not available, skipping pathway diagram.")230 return231 232 G = nx.DiGraph()233 234 # Add all nodes235 all_genes = set()236 for edges in [gt_triplets, model_triplets, prompt_edges]:237 for s, _, t in edges:238 all_genes.add(s)239 all_genes.add(t)240 241 for g in all_genes:242 G.add_node(g)243 244 # Layout245 pos = nx.spring_layout(G, seed=42, k=2.0)246 247 fig, axes = plt.subplots(1, 3, figsize=(18, 6))248 fig.suptitle(tagged_title(title), fontsize=12)249 250 edge_configs = [251 (axes[0], "Prompt Context (Retrieved)", prompt_edges, "#CCCCCC", 1.5),252 (axes[1], "Ground Truth Path", gt_triplets, "#009E73", 2.0),253 (axes[2], "Model Prediction", model_triplets, "#0072B2", 2.0),254 ]255 256 # For model panel, highlight hallucinated edges257 gt_pairs = {(s, t) for s, _, t in gt_triplets}258 prompt_pairs = {(s, t) for s, _, t in prompt_edges}259 260 for ax, panel_title, edges, color, width in edge_configs:261 ax.set_title(panel_title, fontsize=10)262 263 # Draw nodes264 node_colors = []265 for n in G.nodes():266 if n == pert_gene:267 node_colors.append("#FFD700")268 elif n == effect_gene:269 node_colors.append("#FF6B6B")270 else:271 node_colors.append("#E0E0E0")272 273 nx.draw_networkx_nodes(G, pos, ax=ax, node_color=node_colors,274 node_size=800, edgecolors="black", linewidths=1)275 nx.draw_networkx_labels(G, pos, ax=ax, font_size=7)276 277 # Draw edges278 for src, rel, tgt in edges:279 if src not in pos or tgt not in pos:280 continue281 282 edge_color = color283 style = "solid"284 # For model panel, highlight hallucinated285 if panel_title.startswith("Model") and (src, tgt) not in prompt_pairs:286 edge_color = "#D55E00"287 style = "dashed"288 289 arrow_style = "solid" if rel == "activates" else ("dashed" if rel == "inhibits" else "dotted")290 nx.draw_networkx_edges(291 G, pos, edgelist=[(src, tgt)], ax=ax,292 edge_color=edge_color, width=width, style=style,293 arrows=True, arrowstyle="-|>", arrowsize=15,294 connectionstyle="arc3,rad=0.1",295 )296 297 # Edge labels298 edge_labels = {(s, t): rel[:3] for s, rel, t in edges if s in pos and t in pos}299 nx.draw_networkx_edge_labels(G, pos, edge_labels, ax=ax, font_size=6)300 301 ax.axis("off")302 303 # Legend304 legend_items = [305 mpatches.Patch(color="#FFD700", label="Perturbation Gene"),306 mpatches.Patch(color="#FF6B6B", label="Effect Gene"),307 mpatches.Patch(color="#009E73", label="GT Edge"),308 mpatches.Patch(color="#0072B2", label="Model Edge"),309 mpatches.Patch(color="#D55E00", label="Hallucinated"),310 ]311 fig.legend(handles=legend_items, loc="lower center", ncol=5, fontsize=8,312 bbox_to_anchor=(0.5, -0.02))313 314 fig.tight_layout(rect=[0, 0.04, 1, 0.95])315 save_figure(fig, out_path)316 317 318# ══════════════════════════════════════════════════════════════319# Case study output320# ══════════════════════════════════════════════════════════════321 322def write_case_study(case_row, gt_rec, pred_rec, category_info, idx, out_dir: Path):323 """Write a single case study as markdown + pathway diagram."""324 key = resolve_key(case_row)325 split, pt, sid = key326 327 prompt = gt_rec.get("prompt", pred_rec.get("prompt", ""))328 model_output = pred_rec.get("model_output", "")329 gt_label = gt_rec.get("label", case_row.get("gt_answer", ""))330 gt_triplets = gt_rec.get("output", {}).get("triplets_cell_conditioned", [])331 gt_triplets = [tuple(t) for t in gt_triplets]332 333 prompt_edges = parse_prompt_edges(prompt)334 meta = parse_prompt_metadata(prompt)335 model_triplets = extract_triplets_from_text(model_output)336 model_answer = extract_answer_from_text(model_output)337 thinking = extract_thinking_from_text(model_output)338 339 # Build markdown340 lines = [341 f"# Case Study: {category_info['description']}",342 f"## Example {idx + 1}",343 "",344 "### Sample Metadata",345 f"- **Condition:** noisy",346 f"- **Presented Split:** {get_split_display_name(split)}",347 f"- **Raw Split:** {split}",348 f"- **Perturbation Type:** {pt}",349 f"- **Sample ID:** {sid}",350 f"- **Test File:** {get_row_file_name(case_row) or pred_rec.get('source_file', 'N/A')}",351 f"- **Cell Type:** {meta.get('cell_type', 'N/A')}",352 f"- **Perturbation:** {meta.get('perturbation', 'N/A')}",353 f"- **Effect Gene:** {meta.get('effect_gene', 'N/A')}",354 f"- **GT Answer:** {gt_label}",355 f"- **Model Answer:** {model_answer}",356 f"- **Error Label:** {case_row.get('error_label', 'N/A')}",357 f"- **Rescue Verdict:** {case_row.get('llm_rescue_verdict', 'N/A')}",358 "",359 "### Evaluation Metrics",360 f"- Edge F1 (strict): {case_row.get('edge_f1_strict', 'N/A')}",361 f"- Path Connectivity: {case_row.get('path_connectivity', 'N/A')}",362 f"- Sign Match Rate: {case_row.get('sign_match_rate', 'N/A')}",363 f"- Hallucination Rate: {case_row.get('hallucination_rate', 'N/A')}",364 f"- Model Triplets: {case_row.get('num_model_triplets', 'N/A')}",365 "",366 "### Retrieved Knowledge Edges (from prompt)",367 "```",368 ]369 for s, r, t in prompt_edges:370 lines.append(f" {s} --[{r}]--> {t}")371 lines.extend(["```", ""])372 373 lines.extend([374 "### Ground Truth Causal Path",375 "```",376 ])377 for edge in gt_triplets:378 if len(edge) == 3:379 lines.append(f" {edge[0]} --[{edge[1]}]--> {edge[2]}")380 lines.extend(["```", ""])381 382 lines.extend([383 "### Model's Reasoning",384 "```",385 thinking[:2000] if thinking else "(no thinking block)",386 "```",387 "",388 "### Model's Causal Chain",389 "```",390 ])391 for s, r, t in model_triplets:392 lines.append(f" {s} --[{r}]--> {t}")393 lines.extend(["```", ""])394 395 rescue_expl = case_row.get("llm_rescue_explanation", "")396 if rescue_expl:397 lines.extend([398 "### LLM Rescue Explanation",399 f"> {rescue_expl[:1000]}",400 "",401 ])402 403 lines.append(f"")404 405 md_path = out_dir / f"case_{idx}.md"406 md_path.write_text("\n".join(lines))407 print(f" Wrote: {md_path}")408 409 # Render pathway diagram410 pert_gene = meta.get("perturbation", "").split()[-1] if meta.get("perturbation") else ""411 effect_gene = meta.get("effect_gene", "")412 413 render_pathway_diagram(414 gt_triplets=gt_triplets,415 model_triplets=model_triplets,416 prompt_edges=prompt_edges,417 pert_gene=pert_gene,418 effect_gene=effect_gene,419 title=f"{category_info['description']} — {meta.get('perturbation', '?')} → {effect_gene}",420 out_path=out_dir / f"pathway_{idx}",421 )422 423 return {424 "case_idx": idx,425 "condition": "noisy",426 "split": split,427 "presented_split": get_split_display_name(split),428 "pert_type": pt,429 "sample_id": sid,430 "test_file": get_row_file_name(case_row) or pred_rec.get("source_file", ""),431 "cell_type": meta.get("cell_type", ""),432 "perturbation": meta.get("perturbation", ""),433 "effect_gene": effect_gene,434 "gt_answer": gt_label,435 "model_answer": model_answer or "",436 "error_label": case_row.get("error_label", ""),437 "rescue_verdict": case_row.get("llm_rescue_verdict", ""),438 }439 440 441# ══════════════════════════════════════════════════════════════442# Noisy vs Hidden matched pair443# ══════════════════════════════════════════════════════════════444 445def find_matched_pairs(noisy_rows, hidden_rows, n=3, available_keys=None):446 """447 Find samples with same (split, pert_type, sample_id) that have different448 outcomes in noisy vs hidden.449 """450 noisy_idx = {resolve_key(r): r for r in noisy_rows}451 hidden_idx = {resolve_key(r): r for r in hidden_rows}452 453 pairs = []454 for key in noisy_idx:455 if key not in hidden_idx:456 continue457 if available_keys is not None and key not in available_keys:458 continue459 nr = noisy_idx[key]460 hr = hidden_idx[key]461 n_correct = safe_bool(nr.get("answer_correct"))462 h_correct = safe_bool(hr.get("answer_correct"))463 # Interesting: noisy correct but hidden wrong464 if n_correct and not h_correct:465 pairs.append((key, nr, hr))466 467 # Sort by edge_f1 difference (most dramatic)468 pairs.sort(key=lambda x: abs(469 safe_float(x[1].get("edge_f1_strict")) - safe_float(x[2].get("edge_f1_strict"))470 ), reverse=True)471 472 # Select diverse splits473 selected = []474 seen_splits = set()475 for key, nr, hr in pairs:476 if key[0] not in seen_splits and len(selected) < n:477 selected.append((key, nr, hr))478 seen_splits.add(key[0])479 # Fill remaining480 for key, nr, hr in pairs:481 if len(selected) >= n:482 break483 if (key, nr, hr) not in selected:484 selected.append((key, nr, hr))485 486 return selected[:n]487 488 489# ══════════════════════════════════════════════════════════════490# Representativeness statistics491# ══════════════════════════════════════════════════════════════492 493def compute_representativeness_stats(selected_rows, all_rows_in_category):494 """Compute how typical the selected cases are relative to the full category."""495 if not all_rows_in_category or not selected_rows:496 return {}497 498 stats = {}499 for feat in FEATURES:500 all_vals = [safe_float(r.get(feat)) for r in all_rows_in_category]501 sel_vals = [safe_float(r.get(feat)) for r in selected_rows]502 stats[feat] = {503 "category_mean": np.mean(all_vals),504 "category_median": np.median(all_vals),505 "category_std": np.std(all_vals),506 "selected_mean": np.mean(sel_vals),507 }508 stats["category_size"] = len(all_rows_in_category)509 stats["total_samples"] = len(all_rows_in_category)510 return stats511 512 513# ══════════════════════════════════════════════════════════════514# Main515# ══════════════════════════════════════════════════════════════516 517def main():518 args = parse_args()519 set_model_tag(args.model_name)520 out_dir = Path(args.output_dir)521 out_dir.mkdir(parents=True, exist_ok=True)522 523 print("Loading eval CSVs...")524 noisy_rows = load_samples_csv(args.noisy_csv)525 hidden_rows = load_samples_csv(args.hidden_csv)526 527 print("Building GT and prediction indices...")528 gt_noisy_idx = build_gt_index(args.gt_noisy)529 gt_hidden_idx = build_gt_index(args.gt_hidden)530 pred_noisy_idx = build_pred_index(args.pred_noisy)531 pred_hidden_idx = build_pred_index(args.pred_hidden)532 available_noisy_keys = set(gt_noisy_idx) & set(pred_noisy_idx)533 available_pair_keys = set(gt_noisy_idx) & set(gt_hidden_idx) & set(pred_noisy_idx) & set(pred_hidden_idx)534 print(f" GT: {len(gt_noisy_idx)} noisy, {len(gt_hidden_idx)} hidden")535 print(f" Pred: {len(pred_noisy_idx)} noisy, {len(pred_hidden_idx)} hidden")536 537 categories = define_categories()538 summary_rows = []539 540 for cat_name, cat_info in categories.items():541 cat_dir = out_dir / cat_name542 cat_dir.mkdir(parents=True, exist_ok=True)543 544 print(f"\n[Case Study: {cat_name}] {cat_info['description']}")545 546 # Filter candidates from noisy (primary dataset for case studies)547 candidates = [548 r for r in noisy_rows549 if cat_info["filter"](r) and resolve_key(r) in available_noisy_keys550 ]551 print(f" {len(candidates)} candidates found")552 553 if not candidates:554 print(" SKIP: no candidates")555 continue556 557 # Select representative samples558 selected = select_representatives(candidates, n=args.n_per_category)559 stats = compute_representativeness_stats(selected, candidates)560 561 print(f" Selected {len(selected)} representatives "562 f"(from category of {stats.get('category_size', '?')})")563 564 for i, case_row in enumerate(selected):565 key = resolve_key(case_row)566 gt_rec = gt_noisy_idx.get(key, {})567 pred_rec = pred_noisy_idx.get(key, {})568 569 if not gt_rec or not pred_rec:570 print(f" WARNING: GT or pred not found for {key}, skipping")571 continue572 573 info = write_case_study(case_row, gt_rec, pred_rec, cat_info, i, cat_dir)574 info["category"] = cat_name575 info["category_size"] = stats.get("category_size", 0)576 summary_rows.append(info)577 578 # Write representativeness stats579 if stats:580 stats_lines = [f"# Representativeness — {cat_info['description']}\n"]581 stats_lines.append(f"Category size: {stats.get('category_size', 'N/A')} samples\n")582 for feat in FEATURES:583 if feat in stats:584 fs = stats[feat]585 stats_lines.append(586 f"- **{feat}**: category median={fs['category_median']:.3f}, "587 f"selected mean={fs['selected_mean']:.3f}, "588 f"category std={fs['category_std']:.3f}"589 )590 (cat_dir / "representativeness.md").write_text("\n".join(stats_lines))591 592 # ── Noisy vs Hidden matched pairs ──593 print("\n[Case Study: noisy_vs_hidden] Finding matched pairs...")594 matched_dir = out_dir / "noisy_vs_hidden"595 matched_dir.mkdir(parents=True, exist_ok=True)596 597 pairs = find_matched_pairs(598 noisy_rows,599 hidden_rows,600 n=args.n_per_category,601 available_keys=available_pair_keys,602 )603 print(f" Found {len(pairs)} matched pairs")604 605 for i, (key, nr, hr) in enumerate(pairs):606 split, pt, sid = key607 gt_n = gt_noisy_idx.get(key, {})608 gt_h = gt_hidden_idx.get(key, {})609 pred_n = pred_noisy_idx.get(key, {})610 pred_h = pred_hidden_idx.get(key, {})611 612 meta = parse_prompt_metadata(gt_n.get("prompt", ""))613 614 lines = [615 f"# Noisy vs Hidden — Matched Pair {i + 1}",616 "",617 f"**Presented Split:** {get_split_display_name(split)}",618 f"**Raw Split:** {split}",619 f"**Perturbation Type:** {pt}",620 f"**Sample ID:** {sid}",621 f"**Noisy Test File:** {get_row_file_name(nr)}",622 f"**Hidden Test File:** {get_row_file_name(hr)}",623 f"**Cell Type:** {meta.get('cell_type', 'N/A')}",624 f"**Perturbation:** {meta.get('perturbation', 'N/A')}",625 f"**Effect Gene:** {meta.get('effect_gene', 'N/A')}",626 "",627 "## Noisy Condition",628 f"- Answer: {nr.get('model_answer', '?')} (GT: {nr.get('gt_answer', '?')}) — "629 f"{'CORRECT' if safe_bool(nr.get('answer_correct')) else 'WRONG'}",630 f"- Error Label: {nr.get('error_label', 'N/A')}",631 f"- Edge F1 (strict): {nr.get('edge_f1_strict', '?')}",632 f"- Path: {nr.get('path_connectivity', '?')}",633 "",634 "### Model Reasoning (Noisy)",635 "```",636 extract_thinking_from_text(pred_n.get("model_output", ""))[:1500],637 "```",638 "",639 "## Hidden Condition",640 f"- Answer: {hr.get('model_answer', '?')} (GT: {hr.get('gt_answer', '?')}) — "641 f"{'CORRECT' if safe_bool(hr.get('answer_correct')) else 'WRONG'}",642 f"- Error Label: {hr.get('error_label', 'N/A')}",643 f"- Edge F1 (strict): {hr.get('edge_f1_strict', '?')}",644 f"- Path: {hr.get('path_connectivity', '?')}",645 "",646 "### Model Reasoning (Hidden)",647 "```",648 extract_thinking_from_text(pred_h.get("model_output", ""))[:1500],649 "```",650 ]651 652 (matched_dir / f"pair_{i}.md").write_text("\n".join(lines))653 print(f" Wrote: {matched_dir / f'pair_{i}.md'}")654 655 # ── Write master summary ──656 if summary_rows:657 write_markdown_table(summary_rows, out_dir / "case_study_summary.md",658 title="Case Study Summary")659 660 print(f"\nDone! All case studies in: {out_dir}")661 662 663if __name__ == "__main__":664 main()665