diff --git a/.env.example b/.env.example index 4788a31..d9e8183 100644 --- a/.env.example +++ b/.env.example @@ -22,20 +22,20 @@ UNCENSORED_PROJECTOR_FILE=qwen3.8-27b-abliterated/mmproj-Qwen3.8-27B-ABLITERATED VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf FAST_CONTEXT=76800 -FAST_BATCH_SIZE=64 -FAST_UBATCH_SIZE=32 +FAST_BATCH_SIZE=2048 +FAST_UBATCH_SIZE=64 MEDIUM_CONTEXT=160000 MEDIUM_BATCH_SIZE=2048 MEDIUM_UBATCH_SIZE=512 LARGE_CONTEXT=192000 LARGE_BATCH_SIZE=2048 -LARGE_UBATCH_SIZE=128 +LARGE_UBATCH_SIZE=256 ULTRA_CONTEXT=262144 ULTRA_BATCH_SIZE=2048 ULTRA_UBATCH_SIZE=128 UNCENSORED_CONTEXT=80000 UNCENSORED_BATCH_SIZE=2048 -UNCENSORED_UBATCH_SIZE=128 +UNCENSORED_UBATCH_SIZE=256 FAST_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b MEDIUM_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b MEDIUM_TENSOR_SPLIT=85,15 diff --git a/compose.yaml b/compose.yaml index d5e59a9..b291b88 100644 --- a/compose.yaml +++ b/compose.yaml @@ -105,9 +105,9 @@ services: - --threads-batch - "${LLAMA_THREADS_BATCH:-6}" - --batch-size - - "${FAST_BATCH_SIZE:-64}" + - "${FAST_BATCH_SIZE:-2048}" - --ubatch-size - - "${FAST_UBATCH_SIZE:-32}" + - "${FAST_UBATCH_SIZE:-64}" - --parallel - "${FAST_PARALLEL_SLOTS:-1}" - --kv-unified @@ -269,7 +269,7 @@ services: - --batch-size - "${LARGE_BATCH_SIZE:-2048}" - --ubatch-size - - "${LARGE_UBATCH_SIZE:-128}" + - "${LARGE_UBATCH_SIZE:-256}" - --parallel - "${LARGE_PARALLEL_SLOTS:-1}" - --kv-unified @@ -431,7 +431,7 @@ services: - --batch-size - "${UNCENSORED_BATCH_SIZE:-2048}" - --ubatch-size - - "${UNCENSORED_UBATCH_SIZE:-128}" + - "${UNCENSORED_UBATCH_SIZE:-256}" - --parallel - "${UNCENSORED_PARALLEL_SLOTS:-1}" - --kv-unified diff --git a/config/install.env.example b/config/install.env.example index 0cc1426..2f0adaf 100644 --- a/config/install.env.example +++ b/config/install.env.example @@ -80,15 +80,15 @@ DEFAULT_REASONING_EFFORT=off # draft-model artifact is neither downloaded nor passed to llama-server. FAST_CONTEXT=76800 -FAST_BATCH_SIZE=64 -FAST_UBATCH_SIZE=32 +FAST_BATCH_SIZE=2048 +FAST_UBATCH_SIZE=64 MEDIUM_CONTEXT=160000 MEDIUM_BATCH_SIZE=2048 MEDIUM_UBATCH_SIZE=512 MEDIUM_TENSOR_SPLIT=85,15 LARGE_CONTEXT=192000 LARGE_BATCH_SIZE=2048 -LARGE_UBATCH_SIZE=128 +LARGE_UBATCH_SIZE=256 LARGE_TENSOR_SPLIT=86,14 ULTRA_CONTEXT=262144 ULTRA_BATCH_SIZE=2048 @@ -96,7 +96,7 @@ ULTRA_UBATCH_SIZE=128 ULTRA_TENSOR_SPLIT=80,20 UNCENSORED_CONTEXT=80000 UNCENSORED_BATCH_SIZE=2048 -UNCENSORED_UBATCH_SIZE=128 +UNCENSORED_UBATCH_SIZE=256 UNCENSORED_TENSOR_SPLIT=90,10 UNCENSORED_MTP_MAX=2 LLAMA_THREADS=6 diff --git a/docs/PREFILL_BATCH_TUNING_20260915.md b/docs/PREFILL_BATCH_TUNING_20260915.md new file mode 100644 index 0000000..a4c3c52 --- /dev/null +++ b/docs/PREFILL_BATCH_TUNING_20260915.md @@ -0,0 +1,22 @@ +# Prefill-Batch-Tuning vom 15. September 2026 + +Alle fünf Qwen-Profile wurden auf Athena mit demselben kalten Textprompt von +rund 54.000 Tokens getestet. Prompt-Cache-Treffer wurden ausgeschlossen. Als +Zielwert gilt der schnellste stabile Lauf, nicht die größte startbare Zahl. + +| Profil | vorher | getestet | produktiv | Prefill vorher | Prefill produktiv | +|---|---:|---:|---:|---:|---:| +| Fast | 64 / 32 | 128/64, 2048/64, 2048/96, 2048/128 | **2048 / 64** | 806 tok/s | 1.176 tok/s | +| Medium | 2048 / 128 | 2048/512, 2048/1024 | **2048 / 512** | 1.232 tok/s | 1.597 tok/s | +| Large | 2048 / 128 | 2048/256, 2048/512 | **2048 / 256** | 1.231 tok/s | 1.542 tok/s | +| Ultra | 2048 / 128 | 2048/256, 2048/512 | **2048 / 128** | 1.407 tok/s | 1.407 tok/s | +| Uncensored | 2048 / 128 | 2048/256, 2048/512 | **2048 / 256** | 1.166 tok/s | 1.475 tok/s | + +Die Werte sind `Batch / Micro-Batch`. Fast OOMte mit Micro-Batch 96 während +des langen Prompts und mit 128 beim Start. Ultra OOMte bereits beim Start mit +256 und 512. Medium 1024 sowie Large und Uncensored 512 liefen, waren aber +langsamer als der jeweils kleinere produktive Wert und benötigten mehr Reserve. + +Die Messung prüft Prefill und VRAM unter einem langen Textprompt. Änderungen an +Modell, Kontextgröße, Vision-Projektor, Tensor-Split oder llama.cpp-Build +erfordern einen neuen Vergleichstest. diff --git a/platform/llama/README.md b/platform/llama/README.md index 4d4871d..811acd3 100644 --- a/platform/llama/README.md +++ b/platform/llama/README.md @@ -49,3 +49,6 @@ sie alternativ als dokumentierte Referenz für vollständige systemd-Overrides verwendet werden. Der Profilwechsel erfolgt ausschließlich über `platform/scripts/llama-profile`. + +Die produktiven Batch- und Micro-Batch-Werte samt Messungen stehen in +[docs/PREFILL_BATCH_TUNING_20260915.md](../../docs/PREFILL_BATCH_TUNING_20260915.md).