From ba808e105b5af0776f39699868c8fbf5d2397aec Mon Sep 17 00:00:00 2001 From: Mikei386 <44135113+Mikei386@users.noreply.github.com> Date: Sun, 20 Sep 2026 21:25:23 +0200 Subject: [PATCH] Test Medium microbatch 256 against matched 512 control --- ATHENA.md | 2 + docs/INFERENCE_OPTIMIZATION_TODO_20260920.md | 4 + docs/MEDIUM_MICROBATCH_20260920.md | 71 ++++++++ .../medium-microbatch-20260920/README.md | 35 ++++ .../argument-comparison.json | 10 + .../production.json | 76 ++++++++ .../medium-microbatch-20260920/quick.json | 32 ++++ .../restore-verification.json | 61 +++++++ .../results/medium-ub256/config.json.gz | Bin 0 -> 174 bytes .../results/medium-ub256/gpu.json.gz | Bin 0 -> 237 bytes .../results/medium-ub256/loaded.json.gz | Bin 0 -> 3618 bytes .../results/medium-ub256/result.json.gz | Bin 0 -> 3677 bytes .../results/medium-ub256/server-args.json.gz | Bin 0 -> 453 bytes .../results/medium-ub256/server.log.gz | Bin 0 -> 761 bytes .../results/medium-ub512/config.json.gz | Bin 0 -> 171 bytes .../results/medium-ub512/gpu.json.gz | Bin 0 -> 653 bytes .../results/medium-ub512/loaded.json.gz | Bin 0 -> 3619 bytes .../results/medium-ub512/partial.json.gz | Bin 0 -> 8701 bytes .../results/medium-ub512/result.json.gz | Bin 0 -> 8712 bytes .../results/medium-ub512/server-args.json.gz | Bin 0 -> 454 bytes .../results/medium-ub512/server.log.gz | Bin 0 -> 2257 bytes experiments/medium-microbatch-20260920/run.py | 171 ++++++++++++++++++ .../medium-microbatch-20260920/supervise.py | 52 ++++++ .../verify_restore.py | 29 +++ 24 files changed, 543 insertions(+) create mode 100644 docs/MEDIUM_MICROBATCH_20260920.md create mode 100644 experiments/medium-microbatch-20260920/README.md create mode 100644 experiments/medium-microbatch-20260920/argument-comparison.json create mode 100644 experiments/medium-microbatch-20260920/production.json create mode 100644 experiments/medium-microbatch-20260920/quick.json create mode 100644 experiments/medium-microbatch-20260920/restore-verification.json create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub256/config.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub256/gpu.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub256/loaded.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub256/result.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub256/server-args.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub256/server.log.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub512/config.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub512/gpu.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub512/loaded.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub512/partial.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub512/result.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub512/server-args.json.gz create mode 100644 experiments/medium-microbatch-20260920/results/medium-ub512/server.log.gz create mode 100644 experiments/medium-microbatch-20260920/run.py create mode 100644 experiments/medium-microbatch-20260920/supervise.py create mode 100644 experiments/medium-microbatch-20260920/verify_restore.py diff --git a/ATHENA.md b/ATHENA.md index 4a95ed6..1cfbacc 100644 --- a/ATHENA.md +++ b/ATHENA.md @@ -143,3 +143,5 @@ veröffentlicht werden. Bei neuen Modelltests die [gesicherte Qwen-Referenz vom 20.09.2026](benchmarks/athena-qwen38-reference-20260920/README.md) verwenden. **Qwen nicht automatisch erneut benchmarken.** Das Paket enthält sieben Pure-/MIX-Fälle, Originalantworten, feste Requests, Modell-SHA256 und Laufzeit-/GPU-Konfiguration. Neue Kandidaten separat messen; notwendige Abweichungen dokumentieren. Nur bei begründetem Rekalibrierungsbedarf eine neue Referenzversion anlegen, bestehende Ergebnisse unverändert erhalten. Abschluss des 20.09.: [Effizienz-/NVFP4-Prüfung](docs/ATHENA_EFFICIENCY_REVIEW_20260920.md). Pure/MIX bleiben produktiv; Medium startet nach abgefangenem GPU-Pufferfehler ohne Pipeline-Parallelisierung. Bei künftigem Tuning zuerst Speicherreserve und tatsächlichen Durchsatz prüfen. Native NVIDIA-NVFP4-Gewichte wurden nur auf Kapazität geprüft, nicht benchmarked. + +[Microbatch-Folgetest](docs/MEDIUM_MICROBATCH_20260920.md): 256 lädt, unterschreitet aber die VRAM-Reserve (461 MiB frei auf 5080); keine Inferenzmessung mit 256. Produktiv bleibt 512, kein belegter Gewinn und keine automatische Folgeänderung. diff --git a/docs/INFERENCE_OPTIMIZATION_TODO_20260920.md b/docs/INFERENCE_OPTIMIZATION_TODO_20260920.md index 70f5cc4..6963066 100644 --- a/docs/INFERENCE_OPTIMIZATION_TODO_20260920.md +++ b/docs/INFERENCE_OPTIMIZATION_TODO_20260920.md @@ -41,3 +41,7 @@ Ein-Slot-Folgeversuche: [Bericht](DFLASH2_ONE_SLOT_20260920.md). Draft auf 5080 ## Abschlussbewertung und nächste Priorität [Effizienzcheck](ATHENA_EFFICIENCY_REVIEW_20260920.md): Pure/MIX bleiben produktiv. Medium fällt beim Start wegen fehlender Rechenpuffer auf Betrieb ohne Pipeline-Parallelisierung zurück. Ein späterer Punkt-2-Test sollte gezielt Microbatch/Speicherreserve und tatsächlichen Durchsatz vergleichen. Native NVIDIA-NVFP4-Dateien umfassen 20,42 GiB; nur Kapazität geprüft, kein neuer Inferenzversuch. Keine weiteren Tests automatisch eingeplant. + +## Medium-Microbatch-Kurztest + +[512 gegen 256](MEDIUM_MICROBATCH_20260920.md): 256 lädt ohne vorherige Pufferfehler-Meldung, lässt aber nur 461 MiB auf der 5080 frei. Schutzabbruch vor Inferenz; kein Geschwindigkeitsgewinn belegt. 512 wiederhergestellt. Für einen eventuellen Folgetest zuerst mehr 5080-Reserve durch angepassten Split schaffen. diff --git a/docs/MEDIUM_MICROBATCH_20260920.md b/docs/MEDIUM_MICROBATCH_20260920.md new file mode 100644 index 0000000..edcd569 --- /dev/null +++ b/docs/MEDIUM_MICROBATCH_20260920.md @@ -0,0 +1,71 @@ +# Medium: Microbatch 512 gegen 256 — Kurztest + +20.09.2026. **256 lädt ohne den bisherigen Pufferfehler, hat aber zu wenig +Reserve für die festgelegte Testfreigabe. Deshalb keine produktive Umstellung +und noch kein Geschwindigkeitsvergleich für 256.** + +Die echten Medium-Startargumente wurden vor dem Versuch aus Docker übernommen. +Zwischen beiden Testarmen unterschied sich nachweislich nur `--ubatch-size`. +Modell Pure, Kontext 160.000, zwei Slots mit gemeinsamem KV-Pool, Split 85:15, +Vision auf 3060, MTP3, Flash Attention, q4_0-K/V, Batch 2048 und sechs Threads +blieben gleich. Das Testendpoint war isoliert; TTS blieb auf 3060 resident. + +## Ergebnis + +| Prüfung | Microbatch 512 | Microbatch 256 | +|---|---|---| +| Modell geladen | ja | ja | +| Meldung: Rückfall ohne Pipeline-Parallelisierung | ja | nicht aufgetreten | +| 5080 belegt nach Laden | 15.704 MiB | 15.842 MiB | +| 5080 frei nach Laden | 599 MiB | **461 MiB** | +| 3060 belegt nach Laden, inklusive TTS | 10.918 MiB | 10.642 MiB | +| Mindestreserve von 512 MiB erfüllt | ja | **nein** | +| Inferenzaufgaben durchgeführt | ja | **nein, Schutzabbruch vorher** | + +Bei 512 reproduziert sich der bekannte abgefangene Rechenpufferfehler mit +anschließendem Rückfall. Bei 256 fehlt diese Meldung; das Modell lädt erfolgreich. +Die höhere Belegung der 5080 trotz kleinerer Microbatch ist mit einem anderen +Rechenpuffer-/Pipeline-Pfad vereinbar. **Die Abwesenheit der Fehlermeldung allein +beweist jedoch weder aktive Pipeline-Auslastung noch einen Geschwindigkeitsgewinn.** + +Der Abbruch bei 256 war eine Entscheidung des Testtreibers aufgrund der +festgelegten Reserve, kein CUDA-OOM und kein Modellabsturz. Die Grenze wurde +nicht abgesenkt, um doch noch eine Inferenz zu erzwingen. Auch die Reserve ist +keine Garantie, dass jede Langkontext-/Vision-Anfrage passt. + +## Kleiner Kontrolllauf mit 512 + +| Aufgabe | Messwert | +|---|---:| +| Deutsche Ausgabe, 768 Tokens | 57,2 tok/s | +| Code-Ausgabe, 768 Tokens | 74,9 tok/s | +| Prefill, 24.674 Eingabetokens | 1.782,0 tok/s | +| Ausgabe nach dieser Eingabe, 512 Tokens | 54,8 tok/s | +| Drei eingebaute Fakten wiedergefunden | 3/3 | + +Ein Lauf pro Aufgabe; kein umfassender Qualitätstest, kein Vollkontexttest und +kein Test zweier gleichzeitiger Anfragen. Die Vision-Komponente war geladen, +Bilder wurden nicht verarbeitet. TTFT wurde nicht separat gestreamt gemessen. +Die Anfragen verwenden das gespeicherte Text-/Sampling-Protokoll mit deaktiviertem +Promptcache; die Serverkonfiguration einschließlich RAM-Cache blieb unverändert. +Die 512er-Kontrolle ist ein kleiner neuer Vergleichspunkt für das konkrete +Medium-Profil und ersetzt oder verändert die bisherige Qwen-Referenz nicht. + +## Entscheidung und möglicher nächster Schritt + +**512 bleibt vorerst bestehen.** Ein alleiniger Wechsel auf 256 ist auf Basis +dieses Tests nicht freigegeben. Für einen weiteren Versuch müsste zuerst mehr +Reserve auf der 5080 geschaffen werden, etwa durch eine kleine Verschiebung +weiterer Qwen-Schichten auf die 3060. Dann müsste 256 tatsächlich gemessen werden; +auch dieser mögliche Schritt ist keine Zusage für einen Geschwindigkeitsgewinn. +Kein weiterer Test wurde automatisch gestartet oder eingeplant. + +Das ursprüngliche Medium mit Microbatch 512 wurde wiederhergestellt. Medium, +Router, Controller, Gateway und TTS gesund; Router readiness und minimale +Modellantwort geprüft. Keine Xid-, Kernel-Panic-, OOM-Kill- oder +GPU-fallen-off-Meldungen im geprüften Kerneljournal seit Testbeginn. Kein +Host-Neustart und keine Treiber-/Kernel-/Netzwerkänderung. + +Rohdaten, genaue Argumente, GPU-Samples, Logs und Wiederherstellungsnachweis: +`experiments/medium-microbatch-20260920/` im Repository und +`/data/benchmarks/medium-microbatch-20260920/` auf Athena. diff --git a/experiments/medium-microbatch-20260920/README.md b/experiments/medium-microbatch-20260920/README.md new file mode 100644 index 0000000..e3be9da --- /dev/null +++ b/experiments/medium-microbatch-20260920/README.md @@ -0,0 +1,35 @@ +# Medium microbatch quick A/B, 2026-09-20 + +User-authorized first-priority test: microbatch512 versus256, all other Medium +model settings unchanged. Three sequential requests per arm: frozen German and +code decode prompts (768-token caps), then frozen24576-target prefill/recall +(24674 actual prompt tokens,512 output-token cap). No reasoning in these speed +probes; same sampling/seeds as archived requests. Small matched512 control is +necessary because the frozen baseline has different slot/vision/context settings; +this is not a repeat of the broad Qwen benchmark or an overwrite of that baseline. + +Supervisor snapshots actual production image/Args into production.json. Runner +copies them, changes only microbatch and isolated bind address/port, and sets +log verbosity3. Vision remains loaded on3060; context160000 shared by two slots, +Pure model,85:15 split,MTP3, q4_0 target KV, f16 MTP KV, batch2048, six CPU threads, +cache-RAM32768, TTS resident. Requests disable prompt cache for cold measurement. +No concurrency/vision task/full-context validation in this small speed probe. + +Bounded600-second child; production requests drain before stopping the existing +router/controller/Medium. Isolated read-only model mount,26GiB memory cap with no +additional swap allowance,512MiB loaded VRAM reserve,85C/3GiB host RAM guard, +no core dumps. Supervisor restores the same production containers and waits for +healthy status. Kernel/network/driver untouched. Each result directory is unique +and cannot be overwritten. New scripts are not run automatically from checkout. + +Check startup logs for successful allocation versus explicit fallback without +pipeline parallelism. Absence of a fallback line alone is not proof of actual +pipeline utilization; source/verbose logs may be needed to establish activation. +Judge walltime, prompt/decode rates, VRAM and the three-needle recall together. + +## Outcome + +512 control completed;256 loaded without the earlier fallback message but had +only461MiB free on5080, below the512MiB guard. No256 inference was executed. +Args differ only in --ubatch-size. Production512 restored and checked. See +../../docs/MEDIUM_MICROBATCH_20260920.md. Do not claim a256 speedup or quality result. diff --git a/experiments/medium-microbatch-20260920/argument-comparison.json b/experiments/medium-microbatch-20260920/argument-comparison.json new file mode 100644 index 0000000..c1c2cf4 --- /dev/null +++ b/experiments/medium-microbatch-20260920/argument-comparison.json @@ -0,0 +1,10 @@ +{ + "only_model_argument_change": [ + { + "index": 27, + "flag": "--ubatch-size", + "before": "512", + "after": "256" + } + ] +} diff --git a/experiments/medium-microbatch-20260920/production.json b/experiments/medium-microbatch-20260920/production.json new file mode 100644 index 0000000..2f2b1fb --- /dev/null +++ b/experiments/medium-microbatch-20260920/production.json @@ -0,0 +1,76 @@ +{ + "image": "sha256:5e3c12c145b8045e5731b44b6b97033f24b327ae3d4a3fa85ecdd159cc844907", + "args": [ + "--model", + "/models/qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf", + "--mmproj", + "/models/qwen/mmproj-BF16.gguf", + "--mmproj-offload", + "--mmproj-device", + "CUDA1", + "--alias", + "qwen-medium", + "--ctx-size", + "160000", + "--flash-attn", + "on", + "--cache-type-k", + "q4_0", + "--cache-type-v", + "q4_0", + "--cache-prompt", + "--cache-ram", + "32768", + "--threads", + "6", + "--threads-batch", + "6", + "--batch-size", + "2048", + "--ubatch-size", + "512", + "--parallel", + "2", + "--kv-unified", + "--jinja", + "--reasoning", + "auto", + "--reasoning-preserve", + "--host", + "0.0.0.0", + "--port", + "8080", + "--metrics", + "--fit", + "off", + "--n-gpu-layers", + "all", + "--load-mode", + "none", + "--no-ui", + "--temperature", + "1.0", + "--top-p", + "0.95", + "--top-k", + "20", + "--device", + "CUDA0,CUDA1", + "--main-gpu", + "0", + "--split-mode", + "layer", + "--tensor-split", + "85,15", + "--spec-type", + "draft-mtp", + "--spec-draft-n-max", + "3", + "--spec-draft-type-k", + "f16", + "--spec-draft-type-v", + "f16", + "--spec-draft-p-min", + "0.05" + ] +} diff --git a/experiments/medium-microbatch-20260920/quick.json b/experiments/medium-microbatch-20260920/quick.json new file mode 100644 index 0000000..1a29050 --- /dev/null +++ b/experiments/medium-microbatch-20260920/quick.json @@ -0,0 +1,32 @@ +[ + { + "label": "medium-ub512", + "model": "pure", + "ctx": 160000, + "parallel": 2, + "single": false, + "split": "85,15", + "ubatch": 512, + "mtp": 3, + "vision": true, + "prompts": [ + 24576 + ], + "minimum_headroom_mib": 512 + }, + { + "label": "medium-ub256", + "model": "pure", + "ctx": 160000, + "parallel": 2, + "single": false, + "split": "85,15", + "ubatch": 256, + "mtp": 3, + "vision": true, + "prompts": [ + 24576 + ], + "minimum_headroom_mib": 512 + } +] diff --git a/experiments/medium-microbatch-20260920/restore-verification.json b/experiments/medium-microbatch-20260920/restore-verification.json new file mode 100644 index 0000000..7ef5f75 --- /dev/null +++ b/experiments/medium-microbatch-20260920/restore-verification.json @@ -0,0 +1,61 @@ +{ + "checked_at": 1789932247.2439013, + "uptime": "21:24:07 up 3 days, 10:19, 1 user, load average: 0.49, 0.39, 0.47", + "containers": { + "mike-ai-llama-medium": { + "running": true, + "health": "healthy", + "image": "sha256:5e3c12c145b8045e5731b44b6b97033f24b327ae3d4a3fa85ecdd159cc844907", + "started": "2026-09-20T19:23:24.996629585Z" + }, + "mike-ai-router": { + "running": true, + "health": "healthy", + "image": "sha256:0448758bec6968b29263bcac0f8b4682c6d3029d626f7c12334698c11ef096bb", + "started": "2026-09-20T19:23:35.441398885Z" + }, + "mike-ai-profile-controller": { + "running": true, + "health": "healthy", + "image": "sha256:a5f156d94c4921e671fafa524cf9c0fe91e1cbec0113c1f19c136204d63f243f", + "started": "2026-09-20T19:23:35.287332793Z" + }, + "mike-ai-wireguard-gateway": { + "running": true, + "health": "healthy", + "image": "sha256:0d24e93c85a1c420b52b17666ede5fbd4672d92ab8dc28ea4ac5ba144ebc41d0", + "started": "2026-09-17T09:05:07.164533904Z" + }, + "mike-ai-qwen3-tts": { + "running": true, + "health": "healthy", + "image": "sha256:b363a01d08b1bbecbfc3ca6f585368fae2cfdc591f9ecca6643738369f9a9d98", + "started": "2026-09-19T13:47:00.227813668Z" + } + }, + "router": { + "/health": { + "status": "ok", + "router": "alive" + }, + "/ready": { + "status": "ok", + "router": "alive", + "upstream": "ready" + } + }, + "smoke": { + "content": "OK", + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + }, + "kernel_errors": [], + "gpu": "name, memory.used [MiB], memory.total [MiB], temperature.gpu\nNVIDIA GeForce RTX 3060, 10920 MiB, 12288 MiB, 44\nNVIDIA GeForce RTX 5080, 15714 MiB, 16303 MiB, 52", + "disk": "Filesystem Size Used Avail Use% Mounted on\n/dev/nvme0n1p2 868G 187G 637G 23% /\n/dev/nvme1n1p1 916G 821G 49G 95% /data" +} diff --git a/experiments/medium-microbatch-20260920/results/medium-ub256/config.json.gz b/experiments/medium-microbatch-20260920/results/medium-ub256/config.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..ef6f83137e8815f15a8982f69c9231b740754ab6 GIT binary patch literal 174 zcmV;f08#%RiwFP!00002|0Rvh4#FT1gztR{!nr0E1&yy_Oale&CTt=67!%{WyM;aN zd^BNkavd$@Lts)zng*gy+WTe`4P@9}-jRH2 zpIt}ML)m+-a-3u@sz#S(-BVKaXC;z4wfzkl!bY;B+ACrT7)vH}C&`#N%ZsAz(X+4t cR-9L-Z3+Qr1VdIiiQzZ=0i`mkeC7ZE0DK2Z8vp2$Xv?hu89{nK@=AU5fQ{eG>D+KXr07& zx2=^bq@Ym2A%xt2312>L5eRY3mk=`B9h~e6)nRK15g;NYt1X9*e4Ad*dA=x5&s$}3 zT$MNFcsA;fdSd7X$!YDxWI7iF24q;bUe0b)v>I#7YduS{)xHQ6BJ*3~4rvi+e0wL` zJ=y_eRlS8@k!fHrnNrw6Kbj#RFPiqD)nMhm3@XG1-58P1eUtfxZjLp#p}Te2n&Vbv nEAyT0I3lKe2!Ar%3QW_l8g;9G)lHwycCZZY-nN=4P6Pk|!&GUF literal 0 HcmV?d00001 diff --git a/experiments/medium-microbatch-20260920/results/medium-ub256/loaded.json.gz b/experiments/medium-microbatch-20260920/results/medium-ub256/loaded.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..e7645ca94a1580de275f47e9cbfd4c08e1032b3f GIT binary patch literal 3618 zcmV+-4&Ct|iwFP!00002|HT_?bK5rZdwvCuoeu4Zq)4(P+ZoFzQqo+{Uu*)4eeYryfImJV#B!-%)^qX^{)VrBE?5Ac zR>(X*3;Wq(Je^rRH5qbGO~zTmia|Hsz|d$mgr9sUrU?xK2|CV)gdeQ}q_d=fV8vh@ z_$lH$oAyT20%5kGsk_Ft(^)|{jp6$w|GM*q&m$N}ldM1$Cp?T(0fWEF0Yb);>FG@V z{l38BM}C-v&YDp#;XHIge*p`oqcO}sB9J9gnxxD_flklP&rimq^RvPD{A8k7v3xIJ z&MMAIIv;atRzyP-&${~g#kViMAwRJ1dEzqi=K38u8P0~ea44RT_^r`!HW^pLDNm`? z?PxqcJJZHlh_YMLk%l7mCFszSBagj_suffoE#NCCEAr@cvC2UNJA3m;-++u|PQ({;kcng^bo9p#iplk?$dG#ia| zQrp;fX&{-Z(@9v&XzB!1qz;DEY|@^LSwsWLLw#)wg!hMkmRm`?}B zNtV*RTBjb3Q~!c5EC%O zNuoTy7eU4nKmvX2mk^KILKf7Q`YA;4B4C6@o<1J&6c~@=7GA&s z`4HEI#E;4)I*i7oEmdkGq+9_j)sT}VL2Vd+%OpYO^Kw}etA@FyCL?g6iiG+Cc9o6G zVobwF_FK7@fYAg{A})@EbRlr6_SjQ0 z>Lj|CCF*$vPrkQPUG{6($04Z@oBUzm;tyg!hpErd4dT9d$eECR&co%!gQ5^q4?prr zK=N?ercwH^E#q&83+40wFBg*J|5_K)GE(V6?Xkbgg?>iL(t+AMd}#+#6h6^`QnpDe z|LLNpoC-iYFbUSVwB<2BMbPFk7kYa|Q~GL^L`}|OR!2UHiTLXs$sJ2A3=C~3gI#Fp zN6c&O7Pj5kC>oOOlc*qqLg|v!CCI&aPoZN3zB#BBU6Ui%H0YZ&{ruPbZZCV%? z5af}l?a|T~=ffUB!_IBH+N@EfMMQ&T+AJw{CCWj*`Oitfc|7nzwVR>bNRRS$gMvKNlL~z`=dn8$aHxOnuzCZx#{-wz8m51(2+Zjvt^f2wON6|IV(hyHuNU z{ki_}lpNVNH<4ZGK@LlYK~+BO#NO@96}_UM(!cbSWpt+^5DHXL7Gi2|pwS^h2Hmii z=l{BVI+)(zL6w!c=!1yYwsr4?HVtYhCtdr|EmlmmKm*P)3!5#2b(Pc{z zPT9Ra^pKxs9od_;zgpX^0&xd2;9ssW+F=(HIjs(B97G%=9U`k>yGLx*HQ`?Yzi-CC zo~%=%CpmiDa9T8BzZsCXSsZgbSd}SC5DN8;uqJ$PQNYW|yYB zyz5OTT|fZ+KqNbMp_C9OK2M;P{2eMBo+P042r^E-7=kBqw+he= zD8Oyewn!X&p9U}??pZRh+uPJ`rJ*ByIhCOy$-fwj$(CmcDZ_A)_-rYeUvd}Ep~3#J zp}>siqok-$E~No!-5|rdg53M*nrA5lc*I;xzR4COq6@B0cn0?FRHc2vpDE{okQ}JL znjKj=YU;=GV#p7RLP%vZnVTP*E^1U;E7HthPYEXc7T!iM!1^2OD{_r2EAU@FU)r zp-W|=J^{xCIN0!HDrq36chnCs<@EA=EpdLxKotlR__v@JRA``_SMuj(uo*y6bsO>f zfO#vGuRJa@RmuP@B7vlu5XgrVYi^|figjKK4TD?kvg^={60k>0Vv$x^)DdCBYm)LHCcJk90t zEs=L);Bd*-kzLfh=(`@&wHoP@(cT{I*XPgsqxY?vS|S7}U|At=1$kGG=nkW)@I+R9mT%6%H5LswDxt1UP^JXgQ!y zpi$ZpAXV0>EMwcI2q$$=3SBs!h~#w>I4Dxy&G47h=sO z#VP7F1DYKN>m}5ntdoGK>8#e|-4)@v3DajE_gA=Z2QSp1xu*#rVxM-8f(>I{S6PxsqR$b zPM!)~d?}Vb(KBhX%0e)jjMAsXtYho1ZT<@IK&OjX1z0RbP!Fmn8NZ zKo4b&ow@$;Q0#Dc3#YMM?tIclwFUXH2dt#a{Vt^~eLTSP8(sZg@6u0OjQ4RJ7VX&A z_q)SkLA)ayRcs*dWjeaCWSiIU=)ye5a?i5JJvQ4h)IN|WDQrFDpKo7X zR=%M~IUZB+#c#~TLJx>B$$(6QaU5)&+Jlv@RCs74cdNR{>&3yN0R+?kpULolN~dKd zTxjcA5$+h#H4T5>b|Is<^LWbcx}0sgc1K;(aR9NWd`9|)%DLFqpgz2lcggC)LK{&y z{?>uRSbt%!U>{Oq)iQqBQMS^6=@C*rhcsH!7VZ3ISPF*4Z6#*JHtDB|*;af_`4yGV zLeXw&KlKW|X!%g2{WniLd*=MDM(gEi_vJy=ry1@K3>TPswlGY$PYd$8Me7)v{lnh( z2e#gq9CQlxRiFG6;WNWES^66$NS~C=xYTnmKWHPO)^>}Yx;!Wn@mG(Q>GdFFhPd{| zQm9Kt;iwGdlnJl-!m4J;$k+5~~0EXsEytIF)-j>!OijXa)7nvV;&>S!Eb*r@)4bNwj z{$hOY&W1Bq{QHT8FT(=H;?^T#%jF50fp4~sa(YjDC7b{IPX4I1@xM8K?Wud?y>X?A o>dACdOTHwG7ik(>#{a#Mq(=W6Qux0$PadEA7bc^nfVMRN06kOt-~a#s literal 0 HcmV?d00001 diff --git a/experiments/medium-microbatch-20260920/results/medium-ub256/result.json.gz b/experiments/medium-microbatch-20260920/results/medium-ub256/result.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..f63c8feee495ae40b81e211f2bf630d3e01aa167 GIT binary patch literal 3677 zcmV-j4x;fNiwFP!00002|HT_?bK5rZdwvCuoeu4Zq)4(P+d0cK>Gd(WYaW-lo%HGq z2O=SfHAyf8X-kdW-+sFQA0P=zN}B8COH5#~?_KNy(8ouFSS}UJdP+XR-|!XC1qVX+ncQf{ybc;YX_g=`3j=STPs} zev0_croGX$K$tCP>aKC^bXE{fWB5MFzixfu^9TmgBr8zG2@m5`z~HZPfROQIdODMT zzb~-(ksoHEvu4ywI1ioBU%-OtXbkfY2xN(rCMok!pwqMS^ON!D{A@5jKba_2EZ+;5 zvx>8l&c~dZ710pIvtIuE{M+Zx$Pes$p16#>xq3%VhO=QV9Ev9-erq(GO~%!5%2O(J zI~tGA&a`nBqU_dmq@hTC2|Bdo$V2a7R;I%<6D!j*Gb^)`;faBjlWtZX*m zk1c5yq|S;(ETJjb&k-z5!7D1D+;c*G1Z~13my_}Nv>awQ9jbq) zlj(4JHk>suffoE#NCCEAr@cvC2UNJA3m;-++u|PQ({;kcng^bo9p#iplk?$dG#ia| zQrp;fX&{-Z(@9v&XzB!1qz;DEY|@^LSwsWLLw#)wg!hMkmRm`?}B zNtV*RTBjb3Q~#C$rewv^wvkH!kGWJS=S=$4`RNQ|ax@v&yozp|l;5yONKD!ccv}ML z+D+RNH;ly%0VBurg-a8!G48J-o-l`TQBTt<2-Aeo&@d56Hx(@<8cDlrP~DOzq13wd z4<4luczqrPTdU@GRPYFLg9aNxjGAG9l&cUK*wgZj+9D8_jz`ci4p^e^9Ll++@Ayn; zz{;)a19}vc5EC%O zNuoTy7eU4nKmvX2mk?#|V z#h8YV?6-0)0iy|^L|hu7#E6O_Q%GJ6?Bh~*O%A}Cz{C#z8g}jGuI+mR=|bRC?XkyX z)Jb$NOVslUo_ueoy6o4mk3&)+Hu=NA#UI3e3R9n<8^nEapEDu*ocqg*dqp9p9)9GL zfaLzNO{4T-TgKlG7s}`VUoIrc|FtfpWu(%D+GBr}3;m3gr31Bj_|gufD14#=rEHT{ z{?kQGITe6*U=plzY0G1NilEJ7F7)<_ru5Y+iJF|ntd4vX6YG>V{&9)Uq^PO2RSSq237gE6MMHaSM-X4O8?SRmeHMxKqyc}S%|50o+;2US+)q7Nco+t$4o+BB%4oOJC+H&`*%0u6XG%%gp3LMd{B2e-giMVBo> zIA!tMa%<6fZp)#lTH>B0E^on_Zgj z@~$^cv9QXA6)9IN56J5PKp2rO1+_=QmRS&N;XC0Oh?K5jj*6QefgRWPpwS^WY)hyR zStzwAdYG{VWY2^^Ba{hWb^!tO1Ci|5g;GMC_&kAD@^`3kc#?qDBgi=UVhEnd-6}vg zpa8c;+ahuBeHy@oxMRt{Zf{e&m4=S+!S8CR?5*qzuDF;e#u=thX(t@ zh5|F5kCLK3xs(Q^b%PA+3UcSCYo4VL;1P2%`6gSCh%UG~;ThPwQziy=QO3L%xvWNvsN2Cp1r)nfj2C~x++0E^~v{zo)KP<7eBqd$`!1*gH6F-`pnC9{S;yd><~eUt5iA| z3@$Yu$RgnG4GA-mk_98cGVTy)Bru?67M)S@?!}LgT~|Uf^@e@OAYEu$$vInS97#xw zhmdZIvXyX|i6Ko^yBI23CaI+^Dmqhy2dw}b&}fxmdHT(F-@khEopLfAxJMR{o?gDX zB6O)t)F+@mIh57GTPgt{rdE2fAqdpQ%i)v!;ukZzt+1od;mIv2EWqD zn+@DG-eD)OW0(q-Pi)+}TQ-QR;?#H`ttT)Tk8G8rpqz@qSnLv{ z6KIrn1W1*2D$CfmDFRB|trg}wkbP9T@g06>VZz%Q>x^mncpf z1h&+1py@9pxy#nZpt*x8-U_gWyF4@%XiOcS;fD_gSw?e~vT75x+^r2adoD9b*M(TK zNpXsL&46ad!FmZbDC;C3YC5Ymd3Qy4Zo>50$Nd#9+`$VqXzpnOh}fsyqhQ0BSGj_O zzQ9*{dQXLz|I|H-43njwh}~Bx)!BJFUXAwu!L4ZQbwS0fHEN!3@Q$R&yW z2GB!UV`r{^+!s3>-oj}tmph-dQEfqf>;Wt3a=%MyOCJvK{6<&5*Sqx77UO+fhebQ~ z_5JQ}SP<{XMim>#dzp@|E!pNZJi0K?vD~vPa)-@!47CsBNeWvJ`RChLFLRgyc}){} zb}Qddq#Tba_~JL_Vxb4bm}Eev!8i`KPVK=;S1LR-lDk!1?m7l!1M^IoS^ zXQ5~}wV!&0UbK8D(*B#Lojr5@R-^UuwEOa)>eCGO2ZjqwJzE&2+ouJ2-J*32&HiEU z`vY6=OAb1P`l?TUitw4?nk@Yd6QobdW?bsImmjndQER(JPhB1qiTJAr%k+8>GDBQ@ zV=2@nqi|G)a>|6)d|_Dv7Qa(*sz#w3w&(`Sj;s^3d9Bt7s4}MZdbtBky_7VM)T}3*hIGyvlNYO!EP!9t6|RZ=Ss%xw8`3 v;5&W=^ps3yBXWViej+dYZ~jj38e8^%U?HfWB9~c@9v=M{_d_h8BQ^j4UIh(B literal 0 HcmV?d00001 diff --git a/experiments/medium-microbatch-20260920/results/medium-ub256/server-args.json.gz b/experiments/medium-microbatch-20260920/results/medium-ub256/server-args.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..3e920d8e56fd7518d215941394cef4f4b333c5ad GIT binary patch literal 453 zcmV;$0XqI4iwFP!00002|9z89Z`&{ohVS_mMbBYkyRnlV*A4@A9EJe{hM{0=I#yy! zsw8{Ke*F~dBcGjfv7VP?Nb&Wr$-~1VKqo3{AE(E|e0d;Ze`Yx!Y*@I?(ePUW9_6b_uqHX|<;f@F%v#@Lk%s~?DeCfqoaz`CLQf<$j zG?z@FZ|f;r7OS9{MiEwig5rw88co?;X(KzEDoYlzT7M(vF;Akxwhb%J0^SYKb5=8o zSH5L@i{W7V(1J6*aR7TGZX`P{X;K3Pr4iZ>pJvNhVt;X8QYbIwNs`AclTobHaW%}x zSTh91VIzBJ@j%MQ*=d4!bj)aA2e{y|7cQV@@m{Huq|jKi`u=8db4Eyz?vTu$^K-ll zHyz-$9h2#WDLZ6iyS`e$TDHubwK)!RYK?25T#RF@e46H`Tq;M?IZ!FAZ7Cx!bVuz+ vfO~JXJMy`&>$LbNV)$Q#gpToWSEn@&q}D<+a|r)F`91juyPrLiUIYLDRDsxW literal 0 HcmV?d00001 diff --git a/experiments/medium-microbatch-20260920/results/medium-ub256/server.log.gz b/experiments/medium-microbatch-20260920/results/medium-ub256/server.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..0aea0025acb18a7b5be95e4c05100870a62cc1b4 GIT binary patch literal 761 zcmVzkbI_J6b3bljescKJjN{v)%9Za{xvH z6^H{|B?t*0ucol1n=RlS?#%*^7DT{|0zph+m1~D$dI|kPNv*w(2#7JHSn9z$oYqoN z=sLVGrqPoisIaGY=A#sXs;W6Ut%4Th7flm1mh&J^vow8&_cyDU`DvCtcbH33=Cv#- zpS0ghRM2jr7qo93Jg%u2`eQF1%{=y)cu(4EtfB|@>2;F-x@qlxRoQvZ(Z&8aO#ESz z?i7D0s5W4%t)irCq~xvn6hz*cG|D}5ZW2F>Lq7`J!Re+%H`oJC3|rCMp>jpF<><3O zq;WZXxxIo?k-0&LF!ad)sR$~;v@>IHP*5uWE71fK9r|(T6x|kd(T(v0t`X|}xa^oT zDEep_ww)k1#$DSQqA*T?)uh6xiFAjnV!jgmNkweD^XQlN-Qu{i~!HbBbOt>?9Yco>_v4aSAHPc(W(^v^nG zT3f1vC>tm7iSvz@&N%ajVPPxQGz~tRwAiW+ux8OY$Qvn91&B%STdnINg>o(Uq;#<{0kdk Z#c6eBNg?2bU^%+`b$^7n6Fuet003SCO}_vD literal 0 HcmV?d00001 diff --git a/experiments/medium-microbatch-20260920/results/medium-ub512/gpu.json.gz b/experiments/medium-microbatch-20260920/results/medium-ub512/gpu.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a449204c4b58d24496f1348a576ede4f938335a6 GIT binary patch literal 653 zcmV;80&@KyiwFP!00002|LvJgOB_KEhVT0;hCNHsUEh^kAS8InA%qYR50We*#3*q+ zh5Yx7tBDG2v~?z1hdnI2JJqvW{ZRAv+tmk)VfZ<|hGDrrIbAJZ4b?n?urh)LjUQ$2 zI(~e9c`?QhHsAi-wTXN`+T6K(|7rit{_El0>h0N&FRS6h$Ik;5Ez7;zp_dn{uUA7g zRPSD2pRJF+T?Hzl`@P8O^!zHoaqoB5C)dMES>9azwbzxI6mMo$D;l;_Q!3O<&04A5 zCboG$T->Di0B#0Htfl%KxSg={;g<66IZXsxtZW0ef^!%N%>lcXsG5~FZ-KUhI|?Ju zMca3@|I(HMQANDsOtcX@X-jKTtkdIuNaG&DE!@hCqK`L1u(Dek-pwK!OKbbQWeVO5 zVWv3GLirrt-kOzYR^EbrybVh6DD(Kh$#|!z@Ai}DeY^t-g(+ph9cjzE{ovIu+*+Y% zL|78sTC!HR)@Ej%xCzEB8*6xK-cquvNvC?NKpppfWVzKO65x(0)Wl%?11G9G^?nxb zUO5S(H;5hZMS}8fRzjci_6nr# zwODH0#@okjP4V`=zMvCO1oCWWI@9o0Pr$p`_jc~%Ex@A{WYjFY^*^2JOjF)>Pp+TD z+khq5IJG&+uv#j<+?;p-$7sN~ zo+7@pS${k$5M~RSdTSU4$_3#xhVQ5O*DaS^L@Q+M27grpgiNNh(-Zag zy8;W3c$kImno&OyB6LH(&}3l#0fBllrAf+s6zKHq{QT);JRJ>BPo9p>W@ie3`vG%T zaaI!gkdy00G(`D&mp{Jz=gYs7Z`ccwc#OQc`k6c(os4qnP(G;$^v0v}@mV#Tij*ql zjwh2dbDV`Jd~ar=NVx(X_0-5i|6o>Tqca;Tv(wSk#tKOOw9U%&w40SjxxUdNv8)`+ zXG@v|sk>qkOK1uPbR|nuaEl5kcb(82L8CCm<#cjBJwG{}oX`5z9I}M?&>_KS!;vxP zZzGEHZbG9KPLJ%^}Er%J+M*8P; zIvdT-MkftSpas4PDZmcuv^S~ef(jRO!68VtE$(qXT_;SgMc}L1aZYJGJs*w7Cq?jR zQrnn&G*C>{=_D*>G<5?iQx{|EWZIsLSwsWHLw#)w@+*G!dXeI5fL3K-!gi7nZ zuSAqW;0;6+Y(I`ve1|h> z0Bg6Z513Khnz(5UZ83MWpc+Fud(?&?+8fJ^yjPCDGsTLTmjx8PYY1eSwCOPAAtqpm zlSF%#MS(H+yQ`Fhx-M+pgj!G!mir~31tbs&hRm<;@}Y-v4*~lbMTEJf>8;3>flzz< z#4JQ0{5e%(y}x78Ko=~vQpH`OGBn)zH5`l~4eMWk*> zlN*&Pd(T~05B>fV=UevG`uy9?H(&j9D;DQ3zx(lD@2_6`vJmw3x3TM96zqYKV!@-d z)TvQKm2<`3Z5C_gWXRHldySp3s>7kqv{hbUAdLO zUI9B-B58c8!~&drN%C2MCy*CG#u7jQeHfGw55__k)R*!UB6txnLL=WCk3q5e#a)}9JFzG0jItZy&z)C&jW=T*R#@{MQkomk^*2HRHZmG!%T&Ncz&-b#MH-+ zd=ik{U$kkIKJ2LY+u=g_{Qt{^6#2i_g^Y|;x=?%UuX3Ruk+O22HV>cMfi#7WbfA=N z(#n6jsA;DH&<;$JbuMjr%nuQ?dCY_6Uei>*S|w4Nv)I*m);YL0hm+!jgBN)$z&sWDJnpy5zzF0L9a~< z;{t*_lC?cr`Qm)kCurEY<5Zh9sF()P&OHqzG<-uZk{P zf^gdI^`ZOxH0#LOtodr~v$bf&f#%hOMOysmWtZ@)=jC6>sg6%$Wbk~G`1>&w5 z1ADSgiJs)>al>iRg#Bhn-ez$u@MKk`C`l;PH;M;xl&kW%Pc$!m6~(|!cOpAj(VJbG z?((iTO|h`bhZU*NEDy=+06-W~E(Ntm!j@SOY~ec*8HkjwVUEh1K7k#V`_Sl+8@44> z$}Ch`lzq(D60&DPpb^R>S6x5={XnETcBz#RCtM`ZO8yQNjz|*FdIT9KUkt$$d0P$W z1r*@6Xj>!>zE1;~kasK@I_+(0x6;rNKA+0akmOs&VzL!kLdq~)B%Cc3^Go63IW*WG zHWZlgIZBH96iOP9)(bMME65#B*CIX$mIIdRR!>`{! ztS0ZZ>Zw5T{7IVyRI}#q#FJ z)?7Iq@~;baWdit;3?c!NLIDrJ=^BKB=7&IOg`(sq>5K%h`$-G=CxtKR4WX+T~SdPaCjzW?FvRjy#o9c&7G$(f()`WeI!*dc-xSE+P3 z99|eakVPQ88xm$RB@0G?W!xdqNMJz2EGDDm=kLFR?7EVQsW2I<1kO3vBB;7CDY zJcM*xl&wUlObltV+Qm@ODoHI(QPG(qJZJ^jfJUnf%hP{+{o>V|ueFnzz1pEf{f(i|^^D6$l3^oHOs%|22 z7chUN^OY}Drb-#0MI?|^69Tyr7^>0Keo+@mx~hws7k$^gzE&dxGTz&x{rc?LVEnFCQ!9kP!;ux|pw_!Jd;mIv2EWS5 zn+@DG-eD)OW7rCok8Iq!TQ-QR;?#JktS2!Tj~tz&pqz@qSnL9v$2iIz0?h*MNfYq6 zv-PtGKG*#1TGF&4o?xoqyL-UB9o>OFxoOCYdD#Ot{o!zfSX$^ZIW#>W52T*0!N2a6 z9e&eE4mejVb|% za*5CM%$^D{|EYTv6(&ob$lX^c)!BJFUaj{KyIVK(+J<_-o$CdZRh+IXg?iY0ZmK(# zxRa+s7oUrzkMvBMtg;Y{rlPbfLYt8ai=^tM`}$Aa<>u!L4ZQbwS0fKFN!3@Q$R&yW z1~5a}U}tWA+!s3>-ohCymph-bQEfqf>;Wt3a=%L%OCJvK{6<&5*SpNq7VCXnhebQ~ z_5JQ}SP<{XMim>#dzp@|d#cTAcywW&W3^}X$Q?G@G1NYgCn;<_!UFI-D@|q^< z>{h*@NCh5Kh{bQr!$J>;F~xvNgK->e-P(hdu2gtvBzLR2$m_+yqXh)h|DVb5e@dri zC0uChNfGWC(KQW!+IFF$xbt|*>AIY4x^_oh(s2N>r+h~Gmdd%<)}TJTRd>ny!onEQ zIR4gw!&-k~uizX~V%0K!*-^IAfb9`dJ%=<}(iZLfW>^V^#cd^K#5U>2irH3tP5Bj- zPeRddYCrZ0y=eJRr2RKfJA3B*tw!tRY4_zp)u&nR4=fj$X11_Qw@(Z5y2a=in*GDx z_Xm#ImmG8o^~Hev5aBb!HCgfvlVm{3W?bpHpC7alS!=sxUtb;+iTI0$p6&G@WQMr* z#!~7_M(OGd<(3Jr`NFaUEdEi&tr~@H*rFROJF-sD=CxWUpw5`a>*WqG{Za~5)(LZf z1nPFBdv+T$zrjlF_31*$@{w@w`8^m~Eb-F*zIt0)hbTg}m|kQ&@S!wei0>y!O<+@!q&n pMfGI5sU@xm<3*ammi2#c6sht5h7|s9&7+4${{`vQIc2vs008>){*M3v literal 0 HcmV?d00001 diff --git a/experiments/medium-microbatch-20260920/results/medium-ub512/partial.json.gz b/experiments/medium-microbatch-20260920/results/medium-ub512/partial.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..45863dc03c5eed45d7bf705ff098070bb77a28de GIT binary patch literal 8701 zcmVfijrta6irf=Wm5q) zfCA7)U+RM-U}=73Ui1xq@*96_Gqb9@s}F#b?Va6~Cx%2FS(#aRR%L;|eZ^R9Vurl7 z$9{v~;gfBSxeXs{4xd`F(~QSmqg88&PRE;yPCpKK(m083pvmaz@J}}RW?{IzS_x;$B&N>*i-(*3nrYMUHrt_dQX?sA$=h& zP&4$QG02)DFEWX_MyoYY+qr|p*Sac*$fBWojaoi6?u|-UA5@{z?dzQ?RDkk!8I?|d zGb&$6e&de#VJ28pJ~LxG8qK-O12cjR9ECiJKrFIBQgj@(1viBgmYvqHGwk(S!)_z% zLzEC63M6crrYn8Zm1|=EC@|eQcI(yM{KofO?nWg5k>`(=D8$7^pMU{6_-`=K`$N5N zC}Tj<M@)m&$N{;qz!dPHvU@BHl%e&zg&r_6-Zh_&Z0^pZMP_Z%0dmz1)Hzn* zfx_dy>lMwT$V4^oYL%TlGVkQNIW_&ry5aySnDeN-$?1S8pO}I<1Cps>zh`s@MyHjN zD)VL(c}wnw1WD-uX$!z!Poi?il5@X6pgWpc;lvE4g?4N1dI2ACFU;F%p@dPuO{b#6 zB{2<4OsrgbvH;f2yucx@*A6{50>x{3uDz<|M zSw5e{wu#cE!LmH8;$dVtXn;Xa0wGxy4D>Tj7u)`x5EHbuaa3spulq#Q>a|Iiuzbk3Tr zqqVwh>|f`{`6ii<+PCyE{1ySWwL!DhA2+QJo#sv0#E>V~Erw;HUaoeqxzYf_jRTZQ4VmGkcj31k3J zctJF2XMX!flfgh{Ip&;$9>;FP_8CSnp>Iz3&XsA$d|%&X|M+z2CWDD@_ZjS%lfYvE z;0_Wljl_=p*9itL1#*c^m4g#j1f~^&SGk#RAI8FNa6CK zR^Lr|SHu8z6Oh<~UtwQ2V^@|vjYJ59s@(PsGd3=ICrajYn&4zjx3a5#3Vs}n3ahh! zRbcTiR^NlJKfxP>{oO~21SL8jAAa{ytl*nd{2)64$j67}HBuR_lm2ZHA=&@GB|^0P ze^!JPiS;Ke{vw17S{Ts9bZTx$x64qNgSGs7@;h>KwZEp`ajGhmPi z#i2!6fDQS8OBs78nBWJN7eFexh6u+C0^qs}7DrA7<#8t~F>qp=Fm8o=i;IKon>KWW z*F4bbWjD2*X($MPAIoqf*>lc)w({bDrD`}1EIuRTXWj(Qp<(~Qp+L{nLQ0XJ9#I3- zPV5-t3U+Np3onj9!CgK<;~T7iB6IAC6CSO;-Bc-`;Md6W?2s@JzZEmGw6!W9lfjVX zB!Q5yW@MW`6-Bg?Z6!^s9`+4GgI|JMPzIQPS$n`P5M?p^trxR}c?G0QkW&dWXC9uJ zA|T#(G#}QtdH>HS0Jm1doFrPfPD9D z83kms!0&j562vA8&qLkwK=Eu4@{|tt>g2^+;G%D5dXN!I$A~09?Bo>-kBYp&@ih!9iBiF=@-lz!4N_j|`$I?YeVn3rAcmED}1zUj*#5 z1@I~4pb20UlJPJ&S^!ay{4fZukQ8~6&Y2C~FRWPp+k}{hDZf+7vE}&X>8rDggOdwv zIO7h+RRQv;$(|&9#_=J0`TG1qGFV^-PQjX4d@6ap2Ra1%;DU+ESgL8-Lj?ymw!O)c zIdK@VF=v1>dI;Pk*g$1lR7BZNFJFLlori?f8UGN2xlp(wF&irw(MZ$>$6O_0EAz+_ zgPF{{7&4lS)J!E6g~`Q(Rv2rW?mWiu^wH6iS7%2;$kf3NHU{%_^6G+_GqFWY27(LY zz~PZ8DF8RGOv^@-(~$O>V19^!6*$JWmcSQes4aw-@SDWo3?Qgly56gCU+?qlzvYjp&tbBY{%2g6`^i5&lcJm~RB_)QbF)QUd_d{TV*@f8n zZX3GU9;Y#MiLh@fkuP}Ed)fHSsBgx6+tOPc9EIZf%!{l8tj_%r>eD_Q-!i%z1Az-x zBg&%olf3JrxK?9LX55*h{j#^$G~ShBY61v2>{P;R=5n`!3_wOuz?X!)n801)9d-a7 zqbgwe!o+QJ$_9FsoEmGidm$?0PF>h22&a75EdCILM>)#XV9jLQx5a_G8&kgu!uJJ! z+olwE5l=Ab=k^@%&W!G!Ik~!#5BJj?uqY4r-;jzFIyDYO36P3Zt4i=MYiWjG6q0+K zD<+-^yewMOF`t7+10^k`CUzxaRiat60>X|eGlj9j=7d{0BT!EV+yenfIUr9k-Drb? zWL78JnR>lS15((f8P#_ncdfD+-u(|Ps^FH|HtIALF9cUxP7W$Z(AfcW<|MvS1550< z=hlBm%RMYP21Oi1@RkfKh|8xU11n9(pOC{(_u3gnSkkC1%-L?GxtMcR57~4fR!mYx zNxW78X3N2PI!a69Bv`cSthP$JvxWDoApPm%{tOm+@PrMDcvcN?`Ip_JV8SS)+>kj| zh-G?ePKA*F(me{P$;=AE?N=z-*?C!CEAJt;r*0^<1@?eA=L4upayDHll8PYLpthVufWoFnOpl$`XamW0Pz!&B!(mEncf_94IWNQoX!kn5dLO zmTeq!4+`jZWpnS!=hFfzx$4ug7p6tRweLQ{Mtd{7wEvjZmgYw!Dw}kVW6PdGay-N8 z*4o(6hrLd7+!{`LdXJ}%jIc7y=G-q85!WVOfZO2RtVTlUr%IKqe7_@$S_^M;tX!#k zq24%SMOHFh}}0t;aL-|R>p7Ms8c4E5luQ80=IdAMSMGWL`@_?d3(o+SncRQl z&dlNXLu(%Y@FAg5R{7_s#P#S!j9x_3&66ct=}lI@-_r)|cCjygJs?JPbZs!`4m9%6ZmR|Vr?QQnv;bS&!X6~;v|L+nX~WjAqPu|C_JJuv zZLO^vhTiUW2Sa1n2Qtzb)4-ew29?@Ic+Kt$d2Z)6?HX*dgdU0)RD)Irz<0XCUa#HH z-X2b2%#@dnS6RDRUMou)F5g4tbhARsH|qJFnx5D7Bgoy)z_DXqb&BR0VvvY&XkEhy z@UY*^6WO)6caW(WIyu2v7*R^NMU_#g&?gi zJG3p1i{dFQdn6v5G|$l|2QD+`77rqg9XsqPhtMlDqcvN{hz3L!PrTsT45k7-SOvfy z;LL=<4A&%hr{nLD@K zn+AJ!AlCJW5+AjhJ-fI#Wr7_Vdu(y`Gyq1SiCkMge4N5IVdMu%-Zyzn(W!^ng8cWbit6`NEOAF|{B#Vq-IeO@R5pxtR!3aZ_N-nHS?zkmec8 zv|Y?1VFsIXZp~ee1M=GmDLftvuVWj3x8i`#um&2DGL^SY+J-BEHJZx z&DQ|>1xOX{8#Ki3yur@TUwnJ<;=Fl*a)nq?>me`>BTOWLwh*Sk&lxmF#ClNP2u4C= zx(J{vU<;BIo6m7a04o<52+m1?5GQCL<}&%sm|>mI8K9z z-#rHOR*>mS)q3sB`4|YrIhGrOTzGgthgV(}t$z|nYh(l}%b_TLBP9(%xy0OmWshjp@OCXmqcNW5~r{Va4#B?`^RH9j+qXqN-_ z1i0&ggr`^t#lW26F%LnWK*2yR`~?`C;GfXR+4A;xVgM3z!eeoR3o@}ZZHc+(%Tz64 z;}gCB+cO4j5Nu&C2U`+I18~iQX(E=W#ft!SXdTNo49lVxjDf+(r5P~=m)6kOVJ#=gPsq-}(t{205IxBR z!h{GB%8WHVNL~yQ*&t*dL;UAL(-M>mO#*2kRzW6mi&_Wv5nu|lqSA`CJ;XL(;IS(V z5ZHM<=t7imMIl@n{)iX@WS6**%bt0_N^mk_E|^gX*m6KfqBXE%8S*)Kt+_RhNNtWt z8WA|!0>PnFS{z`h39i_QL$s38v`+MhivN z*w67403}8NP#R-!f-4HqhZT`ys`<{CwuU$Y3<0=_$zlw@lE9l|2tC_jg9k5m*_-#7`fdhtI-+o?@Z$*%gJTGbj=4<3txGR_}nCoyG zc3tq6EFlT|W$lqUWlurluFchDjqOmh(A8VJ4d#uny@gHtn#kH!sx&O(7|Wm?a&CX0 zqY*|_0-jKfPS5vkYbls4#lVfO(H#q|dF`;!v(ZP(ZQ`?;wMIuL_jL}0tx3eBM>udz zdlhox=?iZj!hC@v&@N*aj1tY5>{`HLhh!9$BkyT}n}L1%ISdhTjXeSnKp_+>a95Oe zh{XkxXs9@F6Hx+_@E-OY+zzN* zb_~PVV~X$wLU?!I3kio93~l-XgUA>-S2CldD95LX@I?&rK)DHM(U=qm8GGTWG{$LT z@;~Sj7*?8-@g*%0&?$tVGX@|5NEL=XHF3`@7x_c`@;=~`E0rF-Q=kX$(8eop#XLx; zLJ35K62bBlZA&%Pg_vm^&_GAdNZ2ZLZ{8*`o^C zv>Zput>R5!U(s92z?6at1Wpzd;(kxKY`!2pcU9{Px@TfDSpATs0RfQAl5c^*M^_@G zgF%}DCh+mFt2VhgGXF~G#G`9dm}Y=pQ?js6r4yF)3jiukS9)2_y5)T3OHWwtamG?{ z%Cg~{rQaLmzFj1L*3up3zB?s%+A?VMH_fa(aM^O?qMTPXa))}^!OPQGXuo~7*1M0M zJ?VB1PUiac%b%Ry`uxRYeWpKME8mz?I(n%-e90fbFr*t zZy1AKr=$1!gX$USkw<4}X!@|-Z?%VmL9ag?RGcb&elTtjpea-0h+k;-W+v;`t!(s00UXrsJL+|#>9qIv1$LN%{xOhy{ z*1E%fuhZ_fySm;o+TF_2-v9GwGwXF8L&D;z67}fv76Qmah8% z19XezpAgtVFc0^#`!kgZ2h2wI<;_%_BaeYY$f2%f(HKx-Q3B*?;Ke@MIVI^}ZOt&g*~QEJfH%d=jAiMwRkeQrTY3>b&B)I{;-V_VBI)v7O#vnGM4-U%eIjXh zFVBGESrzBCvYg7RbVUr5%_ZLah{Zog7-H^R))aswD^z@e0LBoSfc1s|em9}=0XrmD zFNBnlQswj?N0 z7ZBk&j}(q(dxLAMbKl@wA8hv+en%AZJ#o<+yOCu>(1i~eVMkR}^<17o@bQuv#_wa* z)(%mY@`D6(fyaPyRAJAi0I8us5W22XvQw^3k;5D19w{lPh5HGRC&y>!7o)>hFHc__ zT^v2mPs$;x$OhuISB5`^&0Q9!P`xWqO&NLhy!>|MF(#ebCkUGOw6j}1_5nK=TdW#|dEQf0Xjm`Lycag~2A|v%?MN?iq8EeAt z#H2!8cdKXBWF7mH`Qgx5z#+-rOolxlq*G8dz*l4f1?>fZ+WO$}Sa;_!)V`2P;I@&JJrZW{n zZ%-nzHO^7wilQ7ThKfv>p{TXHiO7n0A-(9|Ms+s%7yX^^+DenL_l~Lzxx1qf%^osC z)MlOJ+~F4TD;86Px;*CrfK~~^*Q~CWIAX>vqI@S@c(FZo>v#r*!Y7}omW+m2{tW%P zlSVfuUO5Jr>ZvY0F@20orb?_Mbt92R?NFF`J|)l%_METAo*7J!@vLEOQeXz4NuqJj(GX2LHQ-AX21b z7@A=rqkLZudEbnUs2F}U5@oHU5qiVX=&~kp$TGOl_O83f)n z6%w*6*gZ!Fg0ZOSN|@=r8l$J`()Nbpbu>hHw1lRH%4J?qHJCtVx0n+NtP=T9GEp!Z zSXt-!1uC3yzK4a%Wg7nc$y&ej|NUgGv+ZQ9^&dQ0d*uWty~C^TJFmSbH$VLFc6c7x z)1N1gj=n!V>R&y1{g+SHI-5?`jK0?D_V01DX7r6tt2>nc>)my<*3-3at2clR=;^Jt zuJ``r(HczdXx;w(j@H^*yVJ%Kvi{I$54(Ej?z6RiM;mrpdaE<+_4@5@yH$C%HqhI5 z9IXw9cOI?jTF2;i+Wl5<2R9B zSM%ZR=V*adD5|RTdxe~Qi;tDDtfHC!D*LmXJGbwwa z+Moq~N0KYchwy%C;c}%e$~EaiRNNQ^Gkl{9?axKP#qoA%9k1ir?H_b^{+LE#VZ`}l zfn{68Dko8KBJT(3(w0_V$5fsWVFA=0I|?DRfracy+yuprxPJ@j3(bLZc-@ZYnVc^4 z)4Om9AKTuafyf6_CcYp1JsVrfH8$d8ISwv-goO=QCT;`U@PgNk@rFvdLH{1o6}*qP z=__sKk%U3F{VsbM0s63Cm|-vi_~ z+rit*OOVa63Hd5S-Iq1nq~as-!7(wYsIfS_)^>r>^V`qvI--iEC zqsK3fMh7oWpB?O{lKj4r_B=YeIM`3c_&t@jBz;d0UcNloPsRAX+#)HtB!Hr#30IdT zXra&7FhGdpbMCt0N|s6&6j7tgT~}OoF7Ye2LaHq~#o|4|NLdpS&*tWsr}xsYkrPy3 zLU-e7X(9rT%m_4QvYMma-CtBu(5}Q<)Z-j|N#uNP2Jo-Td~s4Jd;sfoU9*V8u+_Wz%|8u*9Ck>-DO6 z&P~wb<^@${EM>_E-BU}`bNhMWk_gJ$0D0+6mVhcs&*{+#-c?5=sInRd&*EFO0n)Pi z!R;SlZ0E@Jq(qIg3vkXeSqO(`KEfQ6{x`%OIMrjgSiQ?3=?+p$4TA|}Rz4?%`^pP! z3rkAr>R;AnFsWy_$2m!Nsixx^|4s{O4j}@xW2HfiLOBjRqKhN}6@y#&6f2SaowfEz ztFlbBoPYjQnGUGmuwPCDhHCt{H5K^sWZ+)uKxIO(AtmUlf2yJ)DQInPpnywpFBMR|2qyRm`==Qtq4gkfUn#7<6^LqZ_UEAor(1@F84r%TuFXcyUFYsfn-Z bOs&28EUx^ri87e4K7IAymg*R(S8D(O4#b@{ literal 0 HcmV?d00001 diff --git a/experiments/medium-microbatch-20260920/results/medium-ub512/result.json.gz b/experiments/medium-microbatch-20260920/results/medium-ub512/result.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..fbf98620d22a499159f4d47c64e394eead9ad7f0 GIT binary patch literal 8712 zcmV+jBKO@NiwFP!00002|J6O&avMjI@BWHfC?-JKK)V|^IHn>fijrta6irf=Wm5q) zfCA7)U+RM-U}=73Ui1xq@*96_Gqb9@s}F#b?Va6~Cx%2FS(#aRR%L;|eZ^R9Vurl7 z$9{v~;gfBSxeXs{4xd`F(~QSmqg88&PRE;yPCpKK(m083pvmaz@J}}RW?{IzS_x;$B&N>*i-(*3nrYMUHrt_dQX?sA$=h& zP&4$QG02)DFEWX_MyoYY+qr|p*Sac*$fBWojaoi6?u|-UA5@{z?dzQ?RDkk!8I?|d zGb&$6e&de#VJ28pJ~LxG8qK-O12cjR9ECiJKrFIBQgj@(1viBgmYvqHGwk(S!)_z% zLzEC63M6crrYn8Zm1|=EC@|eQcI(yM{KofO?nWg5k>`(=D8$7^pMU{6_-`=K`$N5N zC}Tj<M@)m&$N{;qz!dPHvU@BHl%e&zg&r_6-Zh_&Z0^pZMP_Z%0dmz1)Hzn* zfx_dy>lMwT$V4^oYL%TlGVkQNIW_&ry5aySnDeN-$?1S8pO}I<1Cps>zh`s@MyHjN zD)VL(c}wnw1WD-uX$!z!Poi?il5@X6pgWpc;lvE4g?4N1dI2ACFU;F%p@dPuO{b#6 zB{2<4OsrgbvH;f2yucx@*A6{50>x{3uDz<|M zSw5e{wu#cE!LmH8;$dVtXn;Xa0wGxy4D>Tj7u)`x5EHbuaa3spulq#Q>a|Iiuzbk3Tr zqqVwh>|f`{`6ii<+PCyE{1ySWwL!DhA2+QJo#sv0#E>V~Erw;HUaoeqxzYf_jRTZQ4VmGkcj31k3J zctJF2XMX!flfgh{Ip&;$9>;FP_8CSnp>Iz3&XsA$d|%&X|M+z2CWDD@_ZjS%lfYvE z;0_Wljl_=p*9itL1#*c^m4g#j1f~^&SGk#RAI8FNa6CK zR^Lr|SHu8z6Oh<~UtwQ2V^@|vjYJ59s@(PsGd3=ICrajYn&4zjx3a5#3Vs}n3ahh! zRbcTiR^NlJKfxP>{oO~21SL8jAAa{ytl*nd{2)64$j67}HBuR_lm2ZHA=&@GB|^0P ze^!JPiS;Ke{vw17S{Ts9bZTx$x64qNgSGs7@;h>KwZEp`ajGhmPi z#i2!6fDQS8OBs78nBWJN7eFexh6u+C0^qs}7DrA7<#8t~F>qp=Fm8o=i;IKon>KWW z*F4bbWjD2*X($MPAIoqf*>lc)w({bDrD`}1EIuRTXWj(Qp<(~Qp+L{nLQ0XJ9#I3- zPV5-t3U+Np3onj9!CgK<;~T7iB6IAC6CSO;-Bc-`;Md6W?2s@JzZEmGw6!W9lfjVX zB!Q5yW@MW`6-Bg?Z6!^s9`+4GgI|JMPzIQPS$n`P5M?p^trxR}c?G0QkW&dWXC9uJ zA|T#(G#}QtdH>HS0Jm1doFrPfPD9D z83kms!0&j562vA8&qLkwK=Eu4@{|tt>g2^+;G%D5dXN!I$A~09?Bo>-kBYp&@ih!9iBiF=@-lz!4N_j|`$I?YeVn3rAcmED}1zUj*#5 z1@I~4pb20UlJPJ&S^!ay{4fZukQ8~6&Y2C~FRWPp+k}{hDZf+7vE}&X>8rDggOdwv zIO7h+RRQv;$(|&9#_=J0`TG1qGFV^-PQjX4d@6ap2Ra1%;DU+ESgL8-Lj?ymw!O)c zIdK@VF=v1>dI;Pk*g$1lR7BZNFJFLlori?f8UGN2xlp(wF&irw(MZ$>$6O_0EAz+_ zgPF{{7&4lS)J!E6g~`Q(Rv2rW?mWiu^wH6iS7%2;$kf3NHU{%_^6G+_GqFWY27(LY zz~PZ8DF8RGOv^@-(~$O>V19^!6*$JWmcSQes4aw-@SDWo3?Qgly56gCU+?qlzvYjp&tbBY{%2g6`^i5&lcJm~RB_)QbF)QUd_d{TV*@f8n zZX3GU9;Y#MiLh@fkuP}Ed)fHSsBgx6+tOPc9EIZf%!{l8tj_%r>eD_Q-!i%z1Az-x zBg&%olf3JrxK?9LX55*h{j#^$G~ShBY61v2>{P;R=5n`!3_wOuz?X!)n801)9d-a7 zqbgwe!o+QJ$_9FsoEmGidm$?0PF>h22&a75EdCILM>)#XV9jLQx5a_G8&kgu!uJJ! z+olwE5l=Ab=k^@%&W!G!Ik~!#5BJj?uqY4r-;jzFIyDYO36P3Zt4i=MYiWjG6q0+K zD<+-^yewMOF`t7+10^k`CUzxaRiat60>X|eGlj9j=7d{0BT!EV+yenfIUr9k-Drb? zWL78JnR>lS15((f8P#_ncdfD+-u(|Ps^FH|HtIALF9cUxP7W$Z(AfcW<|MvS1550< z=hlBm%RMYP21Oi1@RkfKh|8xU11n9(pOC{(_u3gnSkkC1%-L?GxtMcR57~4fR!mYx zNxW78X3N2PI!a69Bv`cSthP$JvxWDoApPm%{tOm+@PrMDcvcN?`Ip_JV8SS)+>kj| zh-G?ePKA*F(me{P$;=AE?N=z-*?C!CEAJt;r*0^<1@?eA=L4upayDHll8PYLpthVufWoFnOpl$`XamW0Pz!&B!(mEncf_94IWNQoX!kn5dLO zmTeq!4+`jZWpnS!=hFfzx$4ug7p6tRweLQ{Mtd{7wEvjZmgYw!Dw}kVW6PdGay-N8 z*4o(6hrLd7+!{`LdXJ}%jIc7y=G-q85!WVOfZO2RtVTlUr%IKqe7_@$S_^M;tX!#k zq24%SMOHFh}}0t;aL-|R>p7Ms8c4E5luQ80=IdAMSMGWL`@_?d3(o+SncRQl z&dlNXLu(%Y@FAg5R{7_s#P#S!j9x_3&66ct=}lI@-_r)|cCjygJs?JPbZs!`4m9%6ZmR|Vr?QQnv;bS&!X6~;v|L+nX~WjAqPu|C_JJuv zZLO^vhTiUW2Sa1n2Qtzb)4-ew29?@Ic+Kt$d2Z)6?HX*dgdU0)RD)Irz<0XCUa#HH z-X2b2%#@dnS6RDRUMou)F5g4tbhARsH|qJFnx5D7Bgoy)z_DXqb&BR0VvvY&XkEhy z@UY*^6WO)6caW(WIyu2v7*R^NMU_#g&?gi zJG3p1i{dFQdn6v5G|$l|2QD+`77rqg9XsqPhtMlDqcvN{hz3L!PrTsT45k7-SOvfy z;LL=<4A&%hr{nLD@K zn+AJ!AlCJW5+AjhJ-fI#Wr7_Vdu(y`Gyq1SiCkMge4N5IVdMu%-Zyzn(W!^ng8cWbit6`NEOAF|{B#Vq-IeO@R5pxtR!3aZ_N-nHS?zkmec8 zv|Y?1VFsIXZp~ee1M=GmDLftvuVWj3x8i`#um&2DGL^SY+J-BEHJZx z&DQ|>1xOX{8#Ki3yur@TUwnJ<;=Fl*a)nq?>me`>BTOWLwh*Sk&lxmF#ClNP2u4C= zx(J{vU<;BIo6m7a04o<52+m1?5GQCL<}&%sm|>mI8K9z z-#rHOR*>mS)q3sB`4|YrIhGrOTzGgthgV(}t$z|nYh(l}%b_TLBP9(%xy0OmWshjp@OCXmqcNW5~r{Va4#B?`^RH9j+qXqN-_ z1i0&ggr`^t#lW26F%LnWK*2yR`~?`C;GfXR+4A;xVgM3z!eeoR3o@}ZZHc+(%Tz64 z;}gCB+cO4j5Nu&C2U`+I18~iQX(E=W#ft!SXdTNo49lVxjDf+(r5P~=m)6kOVJ#=gPsq-}(t{205IxBR z!h{GB%8WHVNL~yQ*&t*dL;UAL(-M>mO#*2kRzW6mi&_Wv5nu|lqSA`CJ;XL(;IS(V z5ZHM<=t7imMIl@n{)iX@WS6**%bt0_N^mk_E|^gX*m6KfqBXE%8S*)Kt+_RhNNtWt z8WA|!0>PnFS{z`h39i_QL$s38v`+MhivN z*w67403}8NP#R-!f-4HqhZT`ys`<{CwuU$Y3<0=_$zlw@lE9l|2tC_jg9k5m*_-#7`fdhtI-+o?@Z$*%gJTGbj=4<3txGR_}nCoyG zc3tq6EFlT|W$lqUWlurluFchDjqOmh(A8VJ4d#uny@gHtn#kH!sx&O(7|Wm?a&CX0 zqY*|_0-jKfPS5vkYbls4#lVfO(H#q|dF`;!v(ZP(ZQ`?;wMIuL_jL}0tx3eBM>udz zdlhox=?iZj!hC@v&@N*aj1tY5>{`HLhh!9$BkyT}n}L1%ISdhTjXeSnKp_+>a95Oe zh{XkxXs9@F6Hx+_@E-OY+zzN* zb_~PVV~X$wLU?!I3kio93~l-XgUA>-S2CldD95LX@I?&rK)DHM(U=qm8GGTWG{$LT z@;~Sj7*?8-@g*%0&?$tVGX@|5NEL=XHF3`@7x_c`@;=~`E0rF-Q=kX$(8eop#XLx; zLJ35K62bBlZA&%Pg_vm^&_GAdNZ2ZLZ{8*`o^C zv>Zput>R5!U(s92z?6at1Wpzd;(kxKY`!2pcU9{Px@TfDSpATs0RfQAl5c^*M^_@G zgF%}DCh+mFt2VhgGXF~G#G`9dm}Y=pQ?js6r4yF)3jiukS9)2_y5)T3OHWwtamG?{ z%Cg~{rQaLmzFj1L*3up3zB?s%+A?VMH_fa(aM^O?qMTPXa))}^!OPQGXuo~7*1M0M zJ?VB1PUiac%b%Ry`uxRYeWpKME8mz?I(n%-e90fbFr*t zZy1AKr=$1!gX$USkw<4}X!@|-Z?%VmL9ag?RGcb&elTtjpea-0h+k;-W+v;`t!(s00UXrsJL+|#>9qIv1$LN%{xOhy{ z*1E%fuhZ_fySm;o+TF_2-v9GwGwXF8L&D;z67}fv76Qmah8% z19XezpAgtVFc0^#`!kgZ2h2wI<;_%_BaeYY$f2%f(HKx-Q3B*?;Ke@MIVI^}ZOt&g*~QEJfH%d=jAiMwRkeQrTY3>b&B)I{;-V_VBI)v7O#vnGM4-U%eIjXh zFVBGESrzBCvYg7RbVUr5%_ZLah{Zog7-H^R))aswD^z@e0LBoSfc1s|em9}=0XrmD zFNBnlQswj?N0 z7ZBk&j}(q(dxLAMbKl@wA8hv+en%AZJ#o<+yOCu>(1i~eVMkR}^<17o@bQuv#_wa* z)(%mY@`D6(fyaPyRAJAi0I8us5W22XvQw^3k;5D19w{lPh5HGRC&y>!7o)>hFHc__ zT^v2mPs$;x$OhuISB5`^&0Q9!P`xWqO&NLhy!>|MF(#ebCkUGOw6j}1_5nK=TdW#|dEQf0Xjm`Lycag~2A|v%?MN?iq8EeAt z#H2!8cdKXBWF7mH`Qgx5z#+-rOolxlq*G8dz*l4f1?>fZ+WO$}Sa;_!)V`2P;I@&JJrZW{n zZ%-nzHO^7wilQ7ThKfv>p{TXHiO7n0A-(9|Ms+s%7yX^^+DenL_l~Lzxx1qf%^osC z)MlOJ+~F4TD;86Px;*CrfK~~^*Q~CWIAX>vqI@S@c(FZo>v#r*!Y7}omW+m2{tW%P zlSVfuUO5Jr>ZvY0F@20orb?_Mbt92R?NFF`J|)l%_METAo*7J!@vLEOQeXz4NuqJj(GX2LHQ-AX21b z7@A=rqkLZudEbnUs2F}U5@oHU5qiVX=&~kp$TGOl_O83f)n z6%w*6*gZ!Fg0ZOSN|@=r8l$J`()Nbpbu>hHw1lRH%4J?qHJCtVx0n+NtP=T9GEp!Z zSXt-!1uC3yzK4a%Wg7nc$y&ej|NUgGv+ZQ9^&dQ0d*uWty~C^TJFmSbH$VLFc6c7x z)1N1gj=n!V>R&y1{g+SHI-5?`jK0?D_V01DX7r6tt2>nc>)my<*3-3at2clR=;^Jt zuJ``r(HczdXx;w(j@H^*yVJ%Kvi{I$54(Ej?z6RiM;mrpdaE<+_4@5@yH$C%HqhI5 z9IXw9cOI?jTF2;i+Wl5<2R9B zSM%ZR=V*adD5|RTdxe~Qi;tDDtfHC!D*LmXJGbwwa z+Moq~N0KYchwy%C;c}%e$~EaiRNNQ^Gkl{9?axKP#qoA%9k1ir?H_b^{+LE#VZ`}l zfn{68Dko8KBJT(3(w0_V$5fsWVFA=0I|?DRfracy+yuprxPJ@j3(bLZc-@ZYnVc^4 z)4Om9AKTuafyf6_CcYp1JsVrfH8$d8ISwv-goO=QCT;`U@PgNk@rFvdLH{1o6}*qP z=__sKk%U3F{VsbM0s63Cm|-vi_~ z+rit*OOVa63Hd5S-Iq1nq~as-!7(wYsIfS_)^>r>^V`qvI--iEC zqsK3fMh7oWpB?O{lKj4r_B=YeIM`3c_&t@jBz;d0UcNloPsRAX+#)HtB!Hr#30IdT zXra&7FhGdpbMCt0N|s6&6j7tgT~}OoF7Ye2LaHq~#o|4|NLdpS&*tWsr}xsYkrPy3 zLU-e7X(9rT%m_4QvYMma-CtBu(5}Q<)Z-j|N#uNP2Jo-Td~s4Jd;sfoU9*V8u+_Wz%|8u*9Ck>-DO6 z&P~wb<^@${EM>_E-BU}`bNhMWk_gJ$0D0+6mVhcs&*{+#-c?5=sInRd&*EFO0n)Pi z!R;SlZ0E@Jq(qIg3vkXeSqO(`KEfQ6{x`%OIMrjgSiQ?3=?+p$4TA|}Rz4?%`^pP! z3rkAr>R;AnFsWy_$2m!Nsixx^|4s{O4j}@xW2HfiLOBjRqKhN}6@y#&6f2SaowfEz ztFlbBoPYjQnGUGmuwPCDhHCt{H5K^sWZ+)uKxIO(AtmUlf2yJ)DQInPpnywpFBMR|2qyRm`==Qtq4gkfUn#7<6^LqZ_UEAor(1@F84r%TuFXcyUFYsfn-Z mOs&28EUx^rDa&9)T2ZhKTBl?5dY!L6ef8gPTq7QMYXAV@Z=pW` literal 0 HcmV?d00001 diff --git a/experiments/medium-microbatch-20260920/results/medium-ub512/server-args.json.gz b/experiments/medium-microbatch-20260920/results/medium-ub512/server-args.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..7f54fc3150ff3e5874d5c0cd4f88cab1bf2ec9ca GIT binary patch literal 454 zcmV;%0XhC3iwFP!00002|9z89kJ>O0hVT4})t;GucfvfGS536W;}0x%^n^e0Xk7p`#3)bi}77AUjGn(N;Z(Mx4>SPaMU1s zMK}EGx8=*v?;}rkyS{c6Y`IPknqD)ILG0ThtApG29Ws<0&i~0&|iC2VZ*mlHO5=_f*?+ zC(SKW=-YbA*Q-s?Orr=ZKf!83VT-10CfdlZrpnW0tkzG(BFlp&QDNJL6-Rs+py#Y+ z6t8^G_#Q)G`_O_jzH^Z%MYSR^MqjlzuF z#U#X>s3~1I%A|TEZ=*7=s#OO>!nzKl$7&n|&(0g&=VeT~2tU+yQEk@i9b($tQKZ6p zx7(*Mx?8uCXjByIqDj-W&_?S8)-QUIQ?1l7!f76EgMGe@(@pz3$#$f@C2!X4-)jB& zOPon+SHinnC7;#mYqctxGCmo<{aFS7{k0vZyIpht#=!|{tfHQ=4C?114qU*Z!^ox$A9Bi3;nL%jOY$-Kk>nZD?J0?3!cZ2lJsLQDd+-Qc(3!p=Wn7of&P7fL z+|rArh|?sCiIb2t#qVn0p@1_LTJ*YeTGnmaAj8Wz+!lGBw!=3h3%a2SqGOyHMYZv$ zu&tR+tEvK(kb)Lm>1H|lO$wV4X13w`_}fopTZZ>Fret;ewI<;%L_OB6a@CM46zsLq zO;Mj2YO&DYyR?4j#B@V`#?;k+K6FSGT=%(2P<$bo)WFPLpDo2i1E_U}# z7PUT)@)9anr_U}2cV*sWZJpF%^;nIO%^7lXPoDDzA{b#(jYt+{4jJ56oqmTAq$G@< zZwP`(!fk7IwQ7m1z-@IPv-<=Usl8}_fM-numFmN9zW%K6t(K6RYAcp&D^rSK>h9>q+` zH=40lg)KI8L#1WTh+~O_4#FqYD0VxnY)c3y|7$aatlY0EPLWAq-oAbJOg{Yb3DFPI zuyRXW@uO9xWN$SMk%|wkAA*Th_X94E;;QO%@&-Fw5IrQR^G_>PKIOil+M3>A*Iyu# zm@{v1U(_UiLYVPV3_I?ER8jZ~*}HbP-0zF!E$$Sf)_2beq-g-H7Lio4tJ13T`h*r|o{!CMpe6VLZVVxpL$q6r z6R)Of6MwkI5%!lxxKm~5Ph8Yt7CFwGy7FReFwtms&j8gqRU);sxOA=FyBln>=!Vib zb37F09Vr;;^%UBxM8bzvJVT+X^J0{`8@K0l@j&TICy7Y zM0qTPoN4c(5DLn@5e2O-3*iZcmxRy?bFPFy@UgFHx+2zVqeQC$wYOR<2D&6$&;W7lNSyGW$QE|iGublTlIQec zAf^`+n-?yI_@X<#BwTdM5X(#53aqmI`Kag?qB#JmRNf_NFu$0q&7CTHc# z2(CmcU)cFh`Z&wv3-V>nh=f%O`nZn7%L2yF1q>L9;g8W*YUP}OdBSj^v^d&FgS4Qj zGg2%NFp2F`p+oVOY=(nLSA|E*H)F>j7~IDZ7ms%(o_8CM>zLYyP#;EI+6gCTW~Os+ zq%30Xqd@&@k#b)e*o2f4J*ifyI#RB0VDqGK?18tKWLG04gtZe=n)Rfxv-G6UmihCZ za;HWQoAlu%rsk?Ip2n==xnBc*VN zn?=*bw^Tm;r?mdN;N6wSdkSZj)zv!Uekvci5>(SxX=jUf=p$-ik zPM|n(qp41=`?$ZO8-vWeYVF1#Q?V0Q= zWVOgw#<~W}#q_i#bx70Bk9$_M?B+VZo|w1X1M?php+!DbLrbc;IB7}kp^u)NrQQGL zT$$LUB^45hN7dnwFEqF2*pdz)>NOx2T2jl^*yQpenQcjx&ru`o2iemTHCkW$LQC2n f>JKC=(+f`29f=n|F2ZYi+LZnuA#(l}q8|VNfS-3# literal 0 HcmV?d00001 diff --git a/experiments/medium-microbatch-20260920/run.py b/experiments/medium-microbatch-20260920/run.py new file mode 100644 index 0000000..dfd5c13 --- /dev/null +++ b/experiments/medium-microbatch-20260920/run.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Bounded, isolated Qwen quantization benchmark. Supervisor restores production.""" +import json, pathlib, subprocess, sys, time, urllib.request, threading, signal +ROOT = pathlib.Path('/data/benchmarks/medium-microbatch-20260920') +NAME = 'mike-ai-microbatch-test' +BASE = 'http://127.0.0.1:5005' +GPU0 = 'GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe' +GPU1 = 'GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b' +IMAGE = 'sha256:5e3c12c145b8045e5731b44b6b97033f24b327ae3d4a3fa85ecdd159cc844907' +MODELS = {'mix':'qwen3.8-27b-iq4-mix/Qwen3.8-27B-IQ4-MIX.gguf', 'pure':'qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf', 'byteshape':'byteshape-qwen38-gpu5/model.gguf'} + +def cmd(*args, check=True, timeout=90): + r = subprocess.run(args, capture_output=True, text=True, timeout=timeout) + if check and r.returncode: raise RuntimeError(str(args[:3])+': '+r.stderr[-2000:]) + return r.stdout + +def api(path, data=None, timeout=900): + req = urllib.request.Request(BASE+path, data=None if data is None else json.dumps(data).encode(), headers={'Content-Type':'application/json'}) + with urllib.request.urlopen(req, timeout=timeout) as r: return json.load(r) + +def save(path, data): + path.write_text(json.dumps(data, indent=2, ensure_ascii=False)+'\n') + +def gpu(): + rows = cmd('nvidia-smi','--query-gpu=name,memory.used,memory.total,temperature.gpu,utilization.gpu','--format=csv,noheader,nounits',timeout=15) + return [dict(zip(['name','used','total','temp','util'], [v.strip() for v in row.split(',')])) for row in rows.splitlines()] + +def health_check(): + rows=gpu() + if any(int(x['temp']) >= 85 for x in rows): raise RuntimeError('GPU temperature limit') + mem = dict((a.split(':')[0],int(a.split()[1])) for a in pathlib.Path('/proc/meminfo').read_text().splitlines()) + if mem['MemAvailable'] < 3*1024*1024: raise RuntimeError('Host RAM reserve below 3 GiB') + return rows + +def chat(prompt, max_tokens=512, effort='none', seed=42, tools=None): + health_check() + p={'model':'qwen-medium','messages':[{'role':'user','content':prompt}], 'max_tokens':max_tokens,'temperature':1.0,'top_p':0.95,'top_k':20,'min_p':0.0,'seed':seed,'reasoning_effort':effort,'cache_prompt':False} + if tools: p.update(tools=tools,tool_choice='auto') + start=time.monotonic(); r=api('/v1/chat/completions',p); r['wall_seconds']=time.monotonic()-start + health_check() + return r + +def prefill(n, seed): + import gzip + payload=json.loads(gzip.decompress(pathlib.Path('/opt/mike-ai/stack/benchmarks/athena-qwen38-reference-20260920/frozen-requests.json.gz').read_bytes()))['prefill-'+str(n)] + r=chat(payload['messages'][0]['content'],payload['max_tokens'],seed=payload['seed']) + content=r['choices'][0]['message'].get('content','') + r['recall']={x:x in content for x in ['RAVEN-417','CEDAR-928','ORBIT-563']} + return r + +def run_case(case): + label=case['label']; out=ROOT/label; out.mkdir(exist_ok=True) + if (out/'result.json').exists(): raise RuntimeError('Refusing to overwrite completed case '+label) + save(out/'config.json',case) + print('START',label,flush=True) + single=case.get('single',False) + production=json.loads((ROOT/'production.json').read_text()) + assert production['image']==IMAGE + args=list(production['args']) + for flag,value in [('--ubatch-size',str(case['ubatch'])),('--host','127.0.0.1'),('--port','5005')]: + args[args.index(flag)+1]=value + args += ['--verbosity','3'] + save(out/'server-args.json',args) + cmd('docker','run','-d','--name',NAME,'--gpus','all','--network','host','--read-only','--tmpfs','/tmp:rw,nosuid,nodev,size=256m','--security-opt','no-new-privileges:true','--cap-drop','ALL','--pids-limit','512','--ulimit','core=0','--memory','26g','--memory-swap','26g','--shm-size','1g','--log-opt','max-size=32m','--log-opt','max-file=1','-e','NVIDIA_VISIBLE_DEVICES='+GPU0+','+GPU1,'-e','NVIDIA_DRIVER_CAPABILITIES=compute,utility','-v','/data/models:/models:ro',IMAGE,*args) + stop=threading.Event(); samples=[] + def monitor(): + while not stop.wait(2): + try: + rows=health_check() + samples.append({'time':time.time(),'gpus':rows}) + except RuntimeError as e: + samples.append({'error':str(e),'aborted':True}) + cmd('docker','stop','-t','10',NAME,check=False) + return + except Exception as e: samples.append({'error':str(e)}) + thread=threading.Thread(target=monitor,daemon=True); thread.start() + result={'case':case,'started':time.time()} + try: + for _ in range(150): + try: + if api('/health',timeout=3).get('status')=='ok': break + except Exception: pass + if cmd('docker','inspect',NAME,'--format','{{.State.Running}}').strip()!='true': raise RuntimeError('Test container exited during load') + time.sleep(2) + else: raise RuntimeError('Startup exceeded 300s') + result['idle_gpu']=health_check(); result['props']=api('/props'); result['slots']=api('/slots') + save(out/'loaded.json',result) + # Added after the 88:12 trial: model loading alone can succeed while + # the first real attention graph still needs more CUDA workspace. + minimum=case.get('minimum_headroom_mib',512) + used_devices=['5080'] if single else ['5080','3060'] + for g in result['idle_gpu']: + if any(device in g['name'] for device in used_devices): + free=int(g['total'])-int(g['used']) + if free=262144: break + previous=context + context=min(262144,context+growth) + final={**case,'ctx':context,'label':case['label']+'-validated-'+str(context),'load_only':False,'prompts':[49152,context-1024]} + return run_case(final) + +if __name__=='__main__': + for case in json.loads(pathlib.Path(sys.argv[1]).read_text()): + if case.get('capacity_search'): capacity_case(case) + else: run_case(case) diff --git a/experiments/medium-microbatch-20260920/supervise.py b/experiments/medium-microbatch-20260920/supervise.py new file mode 100644 index 0000000..0313d0a --- /dev/null +++ b/experiments/medium-microbatch-20260920/supervise.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Stop only existing router/controller/model; always restore the same containers.""" +import json, pathlib, subprocess, sys, time, signal +ROOT=pathlib.Path('/data/benchmarks/medium-microbatch-20260920') +NAMES=['mike-ai-router','mike-ai-profile-controller','mike-ai-llama-medium'] +def run(*args,check=True,timeout=90): + return subprocess.run(args,capture_output=True,text=True,check=check,timeout=timeout) +def stop_signal(*_): raise RuntimeError('Supervisor interrupted') +signal.signal(signal.SIGTERM,stop_signal); signal.signal(signal.SIGINT,stop_signal) +# Refuse if the known production state has changed, or if requests are active. +for name in NAMES: + assert run('docker','inspect',name,'--format','{{.State.Running}}').stdout.strip()=='true',name +for attempt in range(60): + slots=json.loads(run('docker','exec',NAMES[-1],'curl','-fsS','http://127.0.0.1:8080/slots').stdout) + if not any(s['is_processing'] for s in slots): break + if attempt==0: print('WAIT production request active; no interruption',flush=True) + time.sleep(3) +else: raise RuntimeError('Production remained busy for 180s; no services stopped') +assert not run('docker','ps','-q','--filter','name=^mike-ai-microbatch-test$').stdout.strip(),'Existing experiment' +production=json.loads(run('docker','inspect',NAMES[-1]).stdout)[0] +(ROOT/'production.json').write_text(json.dumps({'image':production['Image'],'args':production['Args']},indent=2)+'\n') +child=None +try: + run('docker','stop','-t','30',*NAMES[:2]) + # Drain requests already handed to the model, before unloading it. + for _ in range(120): + slots=json.loads(run('docker','exec',NAMES[-1],'curl','-fsS','http://127.0.0.1:8080/slots').stdout) + if not any(s['is_processing'] for s in slots): break + time.sleep(2) + else: raise RuntimeError('Model did not drain') + run('docker','stop','-t','30',NAMES[-1]) + child=subprocess.Popen(['python3',str(ROOT/'run.py'),sys.argv[1]]) + code=child.wait(timeout=600) + if code: raise RuntimeError('Benchmark failed: '+str(code)) +finally: + if child is not None and child.poll() is None: + child.terminate() + try: child.wait(timeout=20) + except subprocess.TimeoutExpired: child.kill(); child.wait(timeout=10) + run('docker','rm','-f','mike-ai-microbatch-test',check=False) + run('docker','start',NAMES[-1]) + for _ in range(150): + if run('docker','inspect',NAMES[-1],'--format','{{.State.Health.Status}}').stdout.strip()=='healthy': break + time.sleep(2) + else: raise RuntimeError('Restored Medium did not become healthy') + run('docker','start',NAMES[1],NAMES[0]) + for _ in range(60): + statuses=[run('docker','inspect',name,'--format','{{.State.Health.Status}}').stdout.strip() for name in NAMES[:2]] + if all(status=='healthy' for status in statuses): break + time.sleep(2) + else: raise RuntimeError('Restored router/controller did not become healthy') + print('RESTORED existing medium/controller/router; all healthy',flush=True) diff --git a/experiments/medium-microbatch-20260920/verify_restore.py b/experiments/medium-microbatch-20260920/verify_restore.py new file mode 100644 index 0000000..bdf82b6 --- /dev/null +++ b/experiments/medium-microbatch-20260920/verify_restore.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Read-only restoration verification plus a two-token model smoke request.""" +import json,pathlib,re,subprocess,time +ROOT=pathlib.Path('/data/benchmarks/medium-microbatch-20260920') +def run(*args):return subprocess.check_output(args,text=True,timeout=30) +report={'checked_at':time.time(),'uptime':run('uptime').strip(),'containers':{}} +for name in ['mike-ai-llama-medium','mike-ai-router','mike-ai-profile-controller','mike-ai-wireguard-gateway','mike-ai-qwen3-tts']: + d=json.loads(run('docker','inspect',name))[0] + report['containers'][name]={'running':d['State']['Running'],'health':d['State'].get('Health',{}).get('Status'),'image':d['Image'],'started':d['State']['StartedAt']} + assert d['State']['Running'],name + if name in ['mike-ai-llama-medium','mike-ai-router','mike-ai-profile-controller','mike-ai-wireguard-gateway']: + assert d['State'].get('Health',{}).get('Status')=='healthy',name +assert report['containers']['mike-ai-llama-medium']['image']=='sha256:5e3c12c145b8045e5731b44b6b97033f24b327ae3d4a3fa85ecdd159cc844907' +assert not run('docker','ps','-q','--filter','name=^mike-ai-microbatch-test$').strip() +probe='import urllib.request,json; print(json.dumps({p:json.load(urllib.request.urlopen("http://127.0.0.1:8081"+p,timeout=20)) for p in ["/health","/ready"]}))' +report['router']=json.loads(run('docker','exec','mike-ai-router','python','-c',probe)) +payload={'model':'qwen-medium','messages':[{'role':'user','content':'Antworte ausschließlich mit OK.'}],'reasoning_effort':'none','max_tokens':8,'temperature':0} +r=json.loads(run('docker','exec','mike-ai-llama-medium','curl','-fsS','--max-time','20','-H','Content-Type: application/json','--data',json.dumps(payload),'http://127.0.0.1:8080/v1/chat/completions')) +report['smoke']={'content':r['choices'][0]['message'].get('content',''),'usage':r.get('usage')} +assert report['smoke']['content'].strip()=='OK',report['smoke'] +started=min(json.loads(p.read_text())['started'] for p in ROOT.glob('*/result.json')) +journal=run('journalctl','-k','--since','@'+str(int(started)-60),'--no-pager') +pattern=re.compile(r'NVRM.*Xid|oom-kill|Out of memory: Killed process|Kernel panic|GPU has fallen off',re.I) +report['kernel_errors']=[line for line in journal.splitlines() if pattern.search(line)] +report['gpu']=run('nvidia-smi','--query-gpu=name,memory.used,memory.total,temperature.gpu','--format=csv').strip() +report['disk']=run('df','-h','/','/data').strip() +(ROOT/'restore-verification.json').write_text(json.dumps(report,indent=2)+'\n') +print(json.dumps(report,indent=2)) +assert not report['kernel_errors'],'Kernel/GPU errors require review'