Preserve Athena Qwen baseline and document ByteShape comparison

2026-09-20 20:50:08 +02:00
parent 82c50962bf
commit 980f339ea4
99 changed files with 2288 additions and 0 deletions
@@ -0,0 +1,38 @@
+import importlib.util,json,pathlib,subprocess,gzip,hashlib
+root=pathlib.Path('/data/benchmarks/byteshape-20260920')
+spec=importlib.util.spec_from_file_location('bench',root/'run.py'); b=importlib.util.module_from_spec(spec);spec.loader.exec_module(b)
+def run(*args):return subprocess.check_output(args,text=True).strip()
+d=json.loads(run('docker','inspect','mike-ai-llama-medium'))[0]
+ips=[v['IPAddress'] for v in d['NetworkSettings']['Networks'].values() if v.get('IPAddress')]
+b.BASE='http://'+ips[0]+':8080'
+prompts={}
+def capture(text,max_tokens=512,effort='none',seed=42,tools=None):
+ p={'model':'benchmark','messages':[{'role':'user','content':text}],'max_tokens':max_tokens,'temperature':1.0,'top_p':.95,'top_k':20,'min_p':0.,'seed':seed,'reasoning_effort':effort,'cache_prompt':False}
+ if tools:p.update(tools=tools,tool_choice='auto')
+ prompts[current]=p
+ return {'choices':[{'message':{'content':''}}]}
+b.chat=capture
+for n in [4096,24576,49152,56320,60416,75776,261120,103424,109568]:
+ current='prefill-'+str(n); b.prefill(n,42)
+ print(current,flush=True)
+for t in json.loads(pathlib.Path('/opt/mike-ai/stack/dev/QWEN38-FINAL-ACCEPTANCE-v1.json').read_text()):
+ current='quality-'+t['id'];capture(t['prompt'],t['max_tokens'],'medium')
+ if t['id'] in ['i3_code_debugging','i4_capacity_planning','i6_state_vs_configuration']:
+  current='followup-'+t['id'];capture(t['prompt'],8192,'medium',43)
+# Extract literal throughput prompts and tool definition from the measured source.
+import ast
+nodes=ast.walk(ast.parse((root/'run.py').read_text()))
+for node in nodes:
+ if isinstance(node,ast.Assign):
+  for target in node.targets:
+   if isinstance(target,ast.Name) and target.id=='prompts':
+    for i,p in enumerate(ast.literal_eval(node.value)):
+     current='decode-'+str(i);capture(p,768,seed=42+i)
+   if isinstance(target,ast.Name) and target.id=='tool':tool=ast.literal_eval(node.value)
+current='tool';capture('Read the current status of server alpha. Use the provided tool exactly once and do not invent its result.',512,tools=[tool])
+(root/'frozen-requests.json.gz').write_bytes(gzip.compress(json.dumps(prompts,ensure_ascii=False,sort_keys=True).encode(),mtime=0))
+meta={'kernel':run('uname','-r'),'cpu':run('lscpu'),'gpu':run('nvidia-smi','--query-gpu=name,uuid,driver_version,memory.total,power.limit','--format=csv'),'topology':run('nvidia-smi','topo','-m'),'models':{}}
+for key in ['pure','mix','byteshape']:
+ p=pathlib.Path('/data/models')/b.MODELS[key]
+ meta['models'][key]={'path':str(p),'bytes':p.stat().st_size,'sha256':run('sha256sum',str(p)).split()[0]}
+(root/'reference-environment.json').write_text(json.dumps(meta,indent=2)+'\n')