110 lines
2.7 KiB
Bash
Executable File
110 lines
2.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
CONTEXT=${1:-}
|
|
SPLIT=${2:-}
|
|
MODE=${3:-text}
|
|
|
|
if [[ -z $CONTEXT || -z $SPLIT || ! $CONTEXT =~ ^[0-9]+$ || ! $SPLIT =~ ^[0-9]+,[0-9]+$ ]]; then
|
|
echo "Usage: $0 CONTEXT TENSOR_SPLIT {text|vision}" >&2
|
|
exit 2
|
|
fi
|
|
if [[ $MODE != text && $MODE != vision ]]; then
|
|
echo "Mode must be text or vision" >&2
|
|
exit 2
|
|
fi
|
|
|
|
MODEL_DIR=${MODEL_DIR:-/data/models/qwen3.8-27b-dirk}
|
|
MODEL=${MODEL_PATH:-/models/qwen3.8-27b-dirk/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf}
|
|
HOST_MODEL_FILE=${HOST_MODEL_FILE:-$MODEL_DIR/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf}
|
|
MODEL_ALIAS=${MODEL_ALIAS:-qwen-dirk-test}
|
|
PROJECTOR=/models/qwen3.8-27b-dirk/mmproj-F16.gguf
|
|
IMAGE=${LLAMA_IMAGE:-mike-ai/llama.cpp:local}
|
|
NAME=mike-ai-llama-dirk-test
|
|
RESULT_DIR=${RESULT_DIR:-/data/benchmarks/dirk-qwen38}
|
|
|
|
[[ -s "$HOST_MODEL_FILE" ]] || {
|
|
echo "Model file is missing: $HOST_MODEL_FILE" >&2
|
|
exit 1
|
|
}
|
|
if [[ $MODE == vision && ! -s "$MODEL_DIR/mmproj-F16.gguf" ]]; then
|
|
echo "Vision projector is missing" >&2
|
|
exit 1
|
|
fi
|
|
|
|
mapfile -t blockers < <(
|
|
docker ps --format '{{.Names}}' |
|
|
grep -E '^mike-ai-llama-' |
|
|
grep -vE '^mike-ai-llama-dashboard$|^mike-ai-llama-dirk-test$' || true
|
|
)
|
|
if ((${#blockers[@]})); then
|
|
printf 'Refusing to start: production model container(s) still running: %s\n' "${blockers[*]}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
|
install -d -m 0755 "$RESULT_DIR"
|
|
|
|
args=(
|
|
--model "$MODEL"
|
|
--alias "$MODEL_ALIAS"
|
|
--ctx-size "$CONTEXT"
|
|
--flash-attn on
|
|
--cache-type-k q4_0
|
|
--cache-type-v q4_0
|
|
--cache-prompt
|
|
--cache-ram 24576
|
|
--threads 6
|
|
--threads-batch 6
|
|
--batch-size 2048
|
|
--ubatch-size 128
|
|
--parallel 1
|
|
--kv-unified
|
|
--jinja
|
|
--reasoning auto
|
|
--reasoning-budget 8192
|
|
--reasoning-preserve
|
|
--host 127.0.0.1
|
|
--port 5004
|
|
--metrics
|
|
--fit off
|
|
--n-gpu-layers all
|
|
--no-mmap
|
|
--temperature 1.0
|
|
--top-p 0.95
|
|
--top-k 20
|
|
--device CUDA0,CUDA1
|
|
--main-gpu 0
|
|
--split-mode layer
|
|
--tensor-split "$SPLIT"
|
|
--spec-type draft-mtp
|
|
--spec-draft-n-max 3
|
|
--spec-draft-type-k f16
|
|
--spec-draft-type-v f16
|
|
--spec-draft-p-min 0.05
|
|
)
|
|
|
|
if [[ $MODE == vision ]]; then
|
|
args+=(--mmproj "$PROJECTOR" --no-mmproj-offload)
|
|
fi
|
|
|
|
label="ctx${CONTEXT}-split${SPLIT//,/-}-${MODE}"
|
|
docker run -d \
|
|
--name "$NAME" \
|
|
--gpus all \
|
|
--network host \
|
|
--read-only \
|
|
--tmpfs /tmp:rw,noexec,nosuid,nodev,size=256m \
|
|
--security-opt no-new-privileges:true \
|
|
--cap-drop ALL \
|
|
--pids-limit 1024 \
|
|
--log-opt max-size=20m \
|
|
--log-opt max-file=2 \
|
|
-v /data/models:/models:ro \
|
|
-v "$RESULT_DIR":/results \
|
|
--label mike-ai.experiment=dirk-qwen38 \
|
|
--label mike-ai.case="$label" \
|
|
"$IMAGE" "${args[@]}"
|
|
|
|
printf 'Started isolated case %s on http://127.0.0.1:5004\n' "$label"
|