#!/usr/bin/env bash set -Eeuo pipefail CONTEXT=${1:-} SPLIT=${2:-} MODE=${3:-text} if [[ -z $CONTEXT || -z $SPLIT || ! $CONTEXT =~ ^[0-9]+$ || ! $SPLIT =~ ^[0-9]+,[0-9]+$ ]]; then echo "Usage: $0 CONTEXT TENSOR_SPLIT {text|vision}" >&2 exit 2 fi if [[ $MODE != text && $MODE != vision ]]; then echo "Mode must be text or vision" >&2 exit 2 fi MODEL_DIR=${MODEL_DIR:-/data/models/qwen3.8-27b-dirk} MODEL=/models/qwen3.8-27b-dirk/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf PROJECTOR=/models/qwen3.8-27b-dirk/mmproj-F16.gguf IMAGE=${LLAMA_IMAGE:-mike-ai/llama.cpp:local} NAME=mike-ai-llama-dirk-test RESULT_DIR=${RESULT_DIR:-/data/benchmarks/dirk-qwen38} [[ -s "$MODEL_DIR/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf" ]] || { echo "Dirk model is missing; run download-model.sh first" >&2 exit 1 } if [[ $MODE == vision && ! -s "$MODEL_DIR/mmproj-F16.gguf" ]]; then echo "Vision projector is missing" >&2 exit 1 fi mapfile -t blockers < <( docker ps --format '{{.Names}}' | grep -E '^mike-ai-llama-' | grep -vE '^mike-ai-llama-dashboard$|^mike-ai-llama-dirk-test$' || true ) if ((${#blockers[@]})); then printf 'Refusing to start: production model container(s) still running: %s\n' "${blockers[*]}" >&2 exit 1 fi docker rm -f "$NAME" >/dev/null 2>&1 || true install -d -m 0755 "$RESULT_DIR" args=( --model "$MODEL" --alias qwen-dirk-test --ctx-size "$CONTEXT" --flash-attn on --cache-type-k q4_0 --cache-type-v q4_0 --cache-prompt --cache-ram 8192 --threads 6 --threads-batch 6 --batch-size 64 --ubatch-size 32 --parallel 1 --jinja --reasoning auto --reasoning-budget 8192 --host 127.0.0.1 --port 5004 --metrics --fit off --n-gpu-layers all --no-mmap --temperature 1.0 --top-p 0.95 --top-k 20 --device CUDA0,CUDA1 --main-gpu 0 --split-mode layer --tensor-split "$SPLIT" --spec-type draft-mtp --spec-draft-n-max 3 --spec-draft-type-k f16 --spec-draft-type-v f16 ) if [[ $MODE == vision ]]; then args+=(--mmproj "$PROJECTOR" --no-mmproj-offload) fi label="ctx${CONTEXT}-split${SPLIT//,/-}-${MODE}" docker run -d --rm \ --name "$NAME" \ --gpus all \ --network host \ --read-only \ --tmpfs /tmp:rw,noexec,nosuid,nodev,size=256m \ --security-opt no-new-privileges:true \ --cap-drop ALL \ --pids-limit 1024 \ --log-opt max-size=20m \ --log-opt max-file=2 \ -v /data/models:/models:ro \ -v "$RESULT_DIR":/results \ --label mike-ai.experiment=dirk-qwen38 \ --label mike-ai.case="$label" \ "$IMAGE" "${args[@]}" printf 'Started isolated case %s on http://127.0.0.1:5004\n' "$label"