Document Bonsai 2 A/B results

2026-09-19 02:00:40 +02:00
parent de9b17f054
commit 9978e5b7e6
21 changed files with 3141 additions and 6 deletions
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+[[ ${1:-} == --go && $(hostname) == athena ]] || { echo 'Usage on Athena: vision-go.sh --go' >&2; exit 2; }
+cd /opt/mike-ai/experiments/bonsai2-ab
+OUT=/data/benchmarks/bonsai2-ab
+ORIGINAL=$(docker ps --format '{{.Names}}' | sed -n 's/^mike-ai-llama-\(fast\|medium\|large\|ultra\|uncensored\)$/\1/p')
+[[ $(wc -w <<<"$ORIGINAL") == 1 ]] || exit 1
+controller() {
+  docker exec mike-ai-profile-controller python3 -c '
+import os, sys, urllib.request
+token = os.environ.get("CONTROLLER_TOKEN", "").strip()
+if not token:
+    token = open(os.environ.get("CONTROLLER_TOKEN_FILE", "/run/secrets/controller-token"), encoding="utf-8").read().strip()
+req = urllib.request.Request("http://127.0.0.1:8090" + sys.argv[1], data=b"{}", headers={"Authorization": "Bearer " + token, "Content-Type": "application/json"}, method="POST")
+print(urllib.request.urlopen(req, timeout=180).read().decode())
+' "$1"
+}
+finish() {
+  local rc=$?
+  trap - EXIT INT TERM
+  ./case.sh vision stop || true
+  controller "/profiles/$ORIGINAL/activate" || true
+  echo "VISION_EXIT=$rc RESTORED=$ORIGINAL"
+  exit "$rc"
+}
+trap finish EXIT INT TERM
+wait_health() {
+  local url=$1
+  for ((i=0;i<120;i++)); do
+    curl -fsS --max-time 2 "$url/health" >/dev/null 2>&1 && return 0
+    sleep 2
+  done
+  return 1
+}
+controller /profiles/fast/activate
+ip=$(docker inspect mike-ai-llama-fast --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
+wait_health "http://$ip:8080"
+python3 vision_probe.py --base "http://$ip:8080" --model qwen-fast --output "$OUT/vision-qwen-fast.json" --go
+controller /inference/stop
+./case.sh vision create
+./case.sh vision start --go
+wait_health http://127.0.0.1:5006
+nvidia-smi --query-gpu=uuid,memory.used,memory.free --format=csv,noheader,nounits > "$OUT/vision-bonsai-before-gpu.csv"
+python3 vision_probe.py --base http://127.0.0.1:5006 --model bonsai2-test --output "$OUT/vision-bonsai-fast.json" --go
+nvidia-smi --query-gpu=uuid,memory.used,memory.free --format=csv,noheader,nounits > "$OUT/vision-bonsai-after-gpu.csv"
+python3 repeat_critical.py --base http://127.0.0.1:5006 --model bonsai2-test \
+  --temperature 1.0 --seeds 7,42,99 --skip-long \
+  --output "$OUT/recommended-bonsai-fast.json" --go