ajh-code/Jev-Bonsai-Compass

1"""Evaluate supplied labeled decisions; no bundled public benchmark text."""2import argparse,json,statistics,time3from collections import defaultdict4from pathlib import Path5from bonsai_runtime.backend import Client6from bonsai_runtime.decision import DecisionEngine7p=argparse.ArgumentParser();p.add_argument('--input',required=True);p.add_argument('--output',required=True)8p.add_argument('--base',default='http://127.0.0.1:5991');p.add_argument('--profile',choices=['base','current'],default='base');p.add_argument('--adapter-id',type=int,default=0)9a=p.parse_args();engine=DecisionEngine(Client(a.base,None if a.profile=='base' else a.adapter_id));rows=[]10with Path(a.output).open('x') as out:11 for line in Path(a.input).read_text().splitlines():12  item=json.loads(line);request=item['request']13  for order in [0,1]:14   r=dict(request);options=r['options'];r['options']=options[order:]+options[:order]15   result=engine.decide(r);row=dict(id=item['id'],order=order,correct=result['answer']==item['expected'],result=result)16   out.write(json.dumps(row)+'\n');out.flush();rows.append(row)17groups=defaultdict(list)18for r in rows:groups[r['id']].append(r)19canonical=[r for r in rows if r['order']==0]20print(json.dumps(dict(profile=a.profile,requests=len(groups),canonical_correct=sum(r['correct'] for r in canonical),all_order_correct=sum(all(r['correct'] for r in rs) for rs in groups.values()),mean_seconds=statistics.mean(r['result']['seconds'] for r in rows)),indent=2))21