#!/usr/bin/env python3
"""Independent arithmetic check; does not import the metric implementation."""
import json,pathlib,sys
r=pathlib.Path(sys.argv[1]);f=json.load(open(r/'scifact-fixtures.json'));q={x['id']:set(x['relevant']) for x in f['queries']};methods={'capy_recall':{x['id']:x['ranked_ids'] for x in json.load(open(r/'evidence/capy-scifact.json'))},'bm25':json.load(open(r/'evidence/scifact-baseline-rankings.json'))};check={}
for name,preds in methods.items():
 assert set(preds)==set(q);r1=r5=rr=0;hitq=0
 for id,gold in q.items():
  ranked=preds[id];assert len(ranked)==len(set(ranked));r1+=len(set(ranked[:1])&gold)/len(gold);r5+=len(set(ranked[:5])&gold)/len(gold);pos=[i for i,v in enumerate(ranked[:5],1) if v in gold];rr+=1/min(pos) if pos else 0;hitq+=bool(pos)
 check[name]={'n':len(q),'recall_at_1':r1/len(q),'recall_at_5':r5/len(q),'mrr_at_5':rr/len(q),'queries_with_hit_at_5':hitq}
 saved=json.load(open(r/'evidence/scifact-summary.json'))['methods'][name]['scifact_public']
 assert all(abs(check[name][key]-saved[key])<1e-12 for key in ['recall_at_1','recall_at_5','mrr_at_5'])
pred=json.load(open(r/'evidence/routing-predictions-final.json'));s=json.load(open(r/'evidence/routing-summary-final.json'))
for method in ['first_option','always_none','bm25_or_none','bm25_name_description']:
 correct=sum(x[method]==x['gold'] for x in pred);assert correct==s['counts'][method]['correct'];assert sum(s['counts'][method].values())==300
(r/'evidence/independent-final-check.json').write_text(json.dumps(check,indent=2));print(json.dumps(check))