Prepare isolated Dirk Qwen3.8 benchmark

2026-09-01 06:18:39 +02:00
parent b2ea53c383
commit ee28272999
5 changed files with 201 additions and 0 deletions
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+CONTEXT=${1:-}
+SPLIT=${2:-}
+MODE=${3:-text}
+
+if [[ -z $CONTEXT || -z $SPLIT || ! $CONTEXT =~ ^[0-9]+$ || ! $SPLIT =~ ^[0-9]+,[0-9]+$ ]]; then
+  echo "Usage: $0 CONTEXT TENSOR_SPLIT {text|vision}" >&2
+  exit 2
+fi
+if [[ $MODE != text && $MODE != vision ]]; then
+  echo "Mode must be text or vision" >&2
+  exit 2
+fi
+
+MODEL_DIR=${MODEL_DIR:-/data/models/qwen3.8-27b-dirk}
+MODEL=/models/qwen3.8-27b-dirk/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf
+PROJECTOR=/models/qwen3.8-27b-dirk/mmproj-F16.gguf
+IMAGE=${LLAMA_IMAGE:-mike-ai/llama.cpp:local}
+NAME=mike-ai-llama-dirk-test
+RESULT_DIR=${RESULT_DIR:-/data/benchmarks/dirk-qwen38}
+
+[[ -s "$MODEL_DIR/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf" ]] || {
+  echo "Dirk model is missing; run download-model.sh first" >&2
+  exit 1
+}
+if [[ $MODE == vision && ! -s "$MODEL_DIR/mmproj-F16.gguf" ]]; then
+  echo "Vision projector is missing" >&2
+  exit 1
+fi
+
+mapfile -t blockers < <(
+  docker ps --format '{{.Names}}' |
+    grep -E '^mike-ai-llama-' |
+    grep -vE '^mike-ai-llama-dashboard$|^mike-ai-llama-dirk-test$' || true
+)
+if ((${#blockers[@]})); then
+  printf 'Refusing to start: production model container(s) still running: %s\n' "${blockers[*]}" >&2
+  exit 1
+fi
+
+docker rm -f "$NAME" >/dev/null 2>&1 || true
+install -d -m 0755 "$RESULT_DIR"
+
+args=(
+  --model "$MODEL"
+  --alias qwen-dirk-test
+  --ctx-size "$CONTEXT"
+  --flash-attn on
+  --cache-type-k q4_0
+  --cache-type-v q4_0
+  --cache-prompt
+  --cache-ram 8192
+  --threads 6
+  --threads-batch 6
+  --batch-size 64
+  --ubatch-size 32
+  --parallel 1
+  --jinja
+  --reasoning auto
+  --reasoning-budget 8192
+  --host 127.0.0.1
+  --port 5004
+  --metrics
+  --fit off
+  --n-gpu-layers all
+  --no-mmap
+  --temperature 1.0
+  --top-p 0.95
+  --top-k 20
+  --device CUDA0,CUDA1
+  --main-gpu 0
+  --split-mode layer
+  --tensor-split "$SPLIT"
+  --spec-type draft-mtp
+  --spec-draft-n-max 3
+  --spec-draft-type-k f16
+  --spec-draft-type-v f16
+)
+
+if [[ $MODE == vision ]]; then
+  args+=(--mmproj "$PROJECTOR" --no-mmproj-offload)
+fi
+
+label="ctx${CONTEXT}-split${SPLIT//,/-}-${MODE}"
+docker run -d --rm \
+  --name "$NAME" \
+  --gpus all \
+  --network host \
+  --read-only \
+  --tmpfs /tmp:rw,noexec,nosuid,nodev,size=256m \
+  --security-opt no-new-privileges:true \
+  --cap-drop ALL \
+  --pids-limit 1024 \
+  --log-opt max-size=20m \
+  --log-opt max-file=2 \
+  -v /data/models:/models:ro \
+  -v "$RESULT_DIR":/results \
+  --label mike-ai.experiment=dirk-qwen38 \
+  --label mike-ai.case="$label" \
+  "$IMAGE" "${args[@]}"
+
+printf 'Started isolated case %s on http://127.0.0.1:5004\n' "$label"
+