20 lines
1.1 KiB
Python
20 lines
1.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Summarize measured timings; never substitute configured context for input length."""
|
|
import json,pathlib,sys
|
|
root=pathlib.Path(sys.argv[1])
|
|
rows=[]
|
|
for path in sorted(root.glob('*/result.json')):
|
|
r=json.loads(path.read_text()); samples=json.loads((path.parent/'gpu.json').read_text())
|
|
peak={}
|
|
for s in samples:
|
|
for g in s.get('gpus',[]):
|
|
p=peak.setdefault(g['name'],{'MiB':0,'C':0})
|
|
p['MiB']=max(p['MiB'],int(g['used']));p['C']=max(p['C'],int(g['temp']))
|
|
row={'case':r['case'],'finished':bool(r.get('finished')),'peak':peak,'decode':[], 'prefill':[]}
|
|
for d in r.get('decode',[]): row['decode'].append(d.get('timings'))
|
|
for p in r.get('prefill',[]):
|
|
a=p['response'];row['prefill'].append({'target':p['target'],'timings':a.get('timings'),'recall':a.get('recall'),'wall_seconds':a.get('wall_seconds')})
|
|
row['quality']=[{'id':q['id'],'finish':q['response']['choices'][0].get('finish_reason'),'visible_chars':len(q['response']['choices'][0]['message'].get('content','')),'tokens':q['response'].get('usage',{})} for q in r.get('quality',[])]
|
|
rows.append(row)
|
|
print(json.dumps(rows,indent=2,ensure_ascii=False))
|