Tune prefill batching across Qwen profiles

.env.example

@@ -22,12 +22,20 @@ EXPERIMENTAL_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf
 VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf
 
 FAST_CONTEXT=76800
+FAST_BATCH_SIZE=64
+FAST_UBATCH_SIZE=32
 MEDIUM_CONTEXT=160000
 MEDIUM_BATCH_SIZE=2048
 MEDIUM_UBATCH_SIZE=128
 LARGE_CONTEXT=192000
+LARGE_BATCH_SIZE=2048
+LARGE_UBATCH_SIZE=128
 ULTRA_CONTEXT=262144
+ULTRA_BATCH_SIZE=2048
+ULTRA_UBATCH_SIZE=128
 UNCENSORED_CONTEXT=80000
+UNCENSORED_BATCH_SIZE=2048
+UNCENSORED_UBATCH_SIZE=128
 EXPERIMENTAL_CONTEXT=76800
 FAST_GPU_DEVICES=0,1
 MEDIUM_GPU_DEVICES=0,1

compose.yaml

@@ -105,9 +105,9 @@ services:
       - --threads-batch
       - "${LLAMA_THREADS_BATCH:-6}"
       - --batch-size
-      - "64"
+      - "${FAST_BATCH_SIZE:-64}"
       - --ubatch-size
-      - "32"
+      - "${FAST_UBATCH_SIZE:-32}"
       - --parallel
       - "1"
       - --jinja
@@ -268,9 +268,9 @@ services:
       - --threads-batch
       - "${LLAMA_THREADS_BATCH:-6}"
       - --batch-size
-      - "64"
+      - "${LARGE_BATCH_SIZE:-2048}"
       - --ubatch-size
-      - "32"
+      - "${LARGE_UBATCH_SIZE:-128}"
       - --parallel
       - "1"
       - --jinja
@@ -345,9 +345,9 @@ services:
       - --threads-batch
       - "${LLAMA_THREADS_BATCH:-6}"
       - --batch-size
-      - "64"
+      - "${ULTRA_BATCH_SIZE:-2048}"
       - --ubatch-size
-      - "32"
+      - "${ULTRA_UBATCH_SIZE:-128}"
       - --parallel
       - "1"
       - --jinja
@@ -426,9 +426,9 @@ services:
       - --threads-batch
       - "${LLAMA_THREADS_BATCH:-6}"
       - --batch-size
-      - "64"
+      - "${UNCENSORED_BATCH_SIZE:-2048}"
       - --ubatch-size
-      - "32"
+      - "${UNCENSORED_UBATCH_SIZE:-128}"
       - --parallel
       - "1"
       - --jinja

config/install.env.example

@@ -76,15 +76,23 @@ VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c239
 # draft-model artifact is neither downloaded nor passed to llama-server.
 
 FAST_CONTEXT=76800
+FAST_BATCH_SIZE=64
+FAST_UBATCH_SIZE=32
 MEDIUM_CONTEXT=160000
 MEDIUM_BATCH_SIZE=2048
 MEDIUM_UBATCH_SIZE=128
 MEDIUM_TENSOR_SPLIT=90,10
 LARGE_CONTEXT=192000
+LARGE_BATCH_SIZE=2048
+LARGE_UBATCH_SIZE=128
 LARGE_TENSOR_SPLIT=86,14
 ULTRA_CONTEXT=262144
+ULTRA_BATCH_SIZE=2048
+ULTRA_UBATCH_SIZE=128
 ULTRA_TENSOR_SPLIT=80,20
 UNCENSORED_CONTEXT=80000
+UNCENSORED_BATCH_SIZE=2048
+UNCENSORED_UBATCH_SIZE=128
 UNCENSORED_TENSOR_SPLIT=90,10
 UNCENSORED_MTP_MAX=2
 EXPERIMENTAL_CONTEXT=76800

docs/PREFILL_BATCH_BENCHMARK_20260830.md

@@ -0,0 +1,22 @@
+# Qwen prefill batch benchmark (2026-08-30)
+
+Identical 24K-token prompt, one slot, Q4 KV cache and the production GPU split of each profile. The output measurement uses the same deterministic 64-token completion. Full-context checks were run for every changed profile.
+
+| Profile | Context | Previous batch / ubatch | Previous prefill | Previous output | Selected batch / ubatch | Selected prefill | Selected output | Result |
+|---|---:|---:|---:|---:|---:|---:|---:|---|
+| fast | 76,800 | 64 / 32 | 907.2 tok/s | 75.4 tok/s | 64 / 32 | unchanged | unchanged | 2048 / 128 failed with CUDA OOM |
+| medium | 160,000 | 64 / 32 | about 650–710 tok/s | about 65–70 tok/s | 2048 / 128 | 1,585.3 tok/s | 69.6 tok/s | selected; 155K plus vision passed |
+| large | 192,000 | 64 / 32 | 730.6 tok/s | 78.4 tok/s | 2048 / 128 | 1,399.7 tok/s | 76.0 tok/s | selected; 190K plus vision passed |
+| ultra | 262,144 | 64 / 32 | 672.0 tok/s | 57.1 tok/s | 2048 / 128 | 1,649.6 tok/s | 52.7 tok/s | selected; 258K passed |
+| uncensored | 80,000 | 64 / 32 | 729.7 tok/s | 50.8 tok/s | 2048 / 128 | 1,331.8 tok/s | 49.9 tok/s | selected; 78K plus vision passed |
+
+Full-context results with the selected settings:
+
+| Profile | Tested prompt | Prefill | Output | Truncated | Vision after test |
+|---|---:|---:|---:|---|---|
+| medium | 154,998 | 881.3 tok/s | 37.8 tok/s | no | passed |
+| large | 189,998 | 738.0 tok/s | 27.6 tok/s | no | passed |
+| ultra | 257,998 | 698.2 tok/s | 21.1 tok/s | no | not configured for this profile |
+| uncensored | 77,998 | 1,050.5 tok/s | 40.0 tok/s | no | passed |
+
+The larger logical batches 3072 and 4096 did not improve Medium at ubatch 128. Moving Medium from a 90:10 to an 89:11 GPU split freed memory but reduced both prefill and output speed. The production setting therefore remains 2048 / 128 at 90:10.