Prepare isolated Qwen3.8 IQ4_XS A/B tests

2026-09-07 16:22:51 +02:00
parent 1844551534
commit 2ae61baec7
9 changed files with 371 additions and 0 deletions
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+CANDIDATE=${1:-}
+CONTEXT=${2:-}
+SPLIT=${3:-}
+
+if [[ -z $CANDIDATE || -z $CONTEXT || -z $SPLIT || ! $CONTEXT =~ ^[0-9]+$ || ! $SPLIT =~ ^[0-9]+,[0-9]+$ ]]; then
+  echo "Usage: $0 {qwopus|bartowski} CONTEXT TENSOR_SPLIT" >&2
+  exit 2
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=candidates.sh
+source "$SCRIPT_DIR/candidates.sh"
+candidate_config "$CANDIDATE"
+
+HOST_MODEL_FILE=$MODEL_PATH
+CONTAINER_MODEL="/models/$CANDIDATE_ID/$MODEL_FILE"
+IMAGE=${LLAMA_IMAGE:-mike-ai/llama.cpp:local}
+NAME=mike-ai-llama-qwen38-ab
+RESULT_DIR=${RESULT_DIR:-/data/benchmarks/qwen38-ab-20260907}
+
+[[ -s $HOST_MODEL_FILE ]] || {
+  echo "Candidate is not downloaded: $HOST_MODEL_FILE" >&2
+  exit 1
+}
+printf '%s  %s\n' "$MODEL_SHA256" "$HOST_MODEL_FILE" | sha256sum -c -
+
+mapfile -t blockers < <(
+  docker ps --format '{{.Names}}' |
+    grep -E '^mike-ai-llama-' |
+    grep -vE '^mike-ai-llama-dashboard$|^mike-ai-llama-qwen38-ab$' || true
+)
+if ((${#blockers[@]})); then
+  printf 'Refusing to start: production model container(s) still running: %s\n' "${blockers[*]}" >&2
+  exit 1
+fi
+
+if docker ps --format '{{.Names}}' | grep -qx "$NAME"; then
+  echo "Refusing to replace a running A/B container; run ./stop-case.sh first" >&2
+  exit 1
+fi
+
+install -d -m 0755 "$RESULT_DIR"
+
+args=(
+  --model "$CONTAINER_MODEL"
+  --alias "$MODEL_ALIAS"
+  --ctx-size "$CONTEXT"
+  --flash-attn on
+  --cache-type-k q4_0
+  --cache-type-v q4_0
+  --cache-prompt
+  --cache-ram 8192
+  --threads 6
+  --threads-batch 6
+  --batch-size 64
+  --ubatch-size 32
+  --parallel 1
+  --jinja
+  --reasoning auto
+  --reasoning-budget 8192
+  --reasoning-preserve
+  --host 127.0.0.1
+  --port 5005
+  --metrics
+  --fit off
+  --n-gpu-layers all
+  --no-mmap
+  --temperature 1.0
+  --top-p 0.95
+  --top-k 20
+  --device CUDA0,CUDA1
+  --main-gpu 0
+  --split-mode layer
+  --tensor-split "$SPLIT"
+  --spec-type draft-mtp
+  --spec-draft-n-max 3
+  --spec-draft-type-k f16
+  --spec-draft-type-v f16
+)
+
+label="$CANDIDATE-ctx${CONTEXT}-split${SPLIT//,/-}"
+docker run -d \
+  --name "$NAME" \
+  --gpus all \
+  --network host \
+  --read-only \
+  --tmpfs /tmp:rw,noexec,nosuid,nodev,size=256m \
+  --security-opt no-new-privileges:true \
+  --cap-drop ALL \
+  --pids-limit 1024 \
+  --log-opt max-size=20m \
+  --log-opt max-file=2 \
+  -v "$MODEL_ROOT":/models:ro \
+  -v "$RESULT_DIR":/results \
+  --label mike-ai.experiment=qwen38-ab-20260907 \
+  --label mike-ai.candidate="$CANDIDATE" \
+  --label mike-ai.case="$label" \
+  "$IMAGE" "${args[@]}"
+
+printf 'Started isolated case %s on http://127.0.0.1:5005\n' "$label"