michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Document Bonsai 2 A/B results

Browse Source
This commit is contained in:
Mikei386
2026-09-19 02:00:40 +02:00
parent de9b17f054
commit 9978e5b7e6
21 changed files with 3141 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/bonsai2-ab/case.sh
+11 -3
View File
@@ -8,8 +8,8 @@ IMAGE=mike-ai/bonsai2-ab:prism-b10685
MODEL=/data/models/bonsai2-ab/Ternary-Bonsai-2-27B-PQ2_0.gguf
GPU_5080=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe
GPU_3060=GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
[[ $CASE =~ ^(fast|medium|ultra)$ && $ACTION =~ ^(create|start|stop)$ ]] || {
echo 'Usage: case.sh {fast|medium|ultra} {create|start|stop}' >&2; exit 2;
[[ $CASE =~ ^(fast|medium|ultra|vision)$ && $ACTION =~ ^(create|start|stop)$ ]] || {
echo 'Usage: case.sh {fast|medium|ultra|vision} {create|start|stop}' >&2; exit 2;
}
test "$(hostname)" = athena || exit 2
if [[ $ACTION == stop ]]; then
@@ -36,7 +36,15 @@ case "$CASE" in
fast) CONTEXT=76800; UBATCH=64; DEVICES="$GPU_5080"; SPLIT=(--device CUDA0 --split-mode none) ;;
medium) CONTEXT=160000; UBATCH=512; DEVICES="$GPU_5080,$GPU_3060"; SPLIT=(--device CUDA0,CUDA1 --main-gpu 0 --split-mode layer --tensor-split 85,15) ;;
ultra) CONTEXT=262144; UBATCH=128; DEVICES="$GPU_5080,$GPU_3060"; SPLIT=(--device CUDA0,CUDA1 --main-gpu 0 --split-mode layer --tensor-split 80,20) ;;
vision) CONTEXT=76800; UBATCH=64; DEVICES="$GPU_5080,$GPU_3060"; SPLIT=(--device CUDA0 --split-mode none) ;;
esac
VISION=()
if [[ $CASE == vision ]]; then
test -s /data/models/bonsai2-ab/Ternary-Bonsai-2-27B-mmproj-Q8_0.gguf || {
echo 'Verified vision projector is missing' >&2; exit 1;
}
VISION=(--mmproj /models/Ternary-Bonsai-2-27B-mmproj-Q8_0.gguf --mmproj-offload --mmproj-device CUDA1)
fi
docker create --name "$NAME" --gpus "\"device=$DEVICES\"" \
--network bridge -p 127.0.0.1:5006:8080 \
--read-only --tmpfs /tmp:rw,noexec,nosuid,nodev,size=256m \
@@ -50,5 +58,5 @@ docker create --name "$NAME" --gpus "\"device=$DEVICES\"" \
--cache-prompt --threads 6 --threads-batch 6 --batch-size 2048 \
--ubatch-size "$UBATCH" --parallel 1 --jinja --reasoning auto \
--host 0.0.0.0 --port 8080 --metrics --fit off --n-gpu-layers all \
--temperature 1.0 --top-p 0.95 --top-k 20 "${SPLIT[@]}" >/dev/null
--temperature 1.0 --top-p 0.95 --top-k 20 "${VISION[@]}" "${SPLIT[@]}" >/dev/null
echo "Created stopped Bonsai $CASE case ($CONTEXT context)."