Tune prefill batching across Qwen profiles

2026-08-30 22:11:26 +02:00
parent 9c10b0ab88
commit 27d2bb1b4b
4 changed files with 46 additions and 8 deletions
@@ -105,9 +105,9 @@ services:
      - --threads-batch
      - "${LLAMA_THREADS_BATCH:-6}"
      - --batch-size
-      - "64"
+      - "${FAST_BATCH_SIZE:-64}"
      - --ubatch-size
-      - "32"
+      - "${FAST_UBATCH_SIZE:-32}"
      - --parallel
      - "1"
      - --jinja
@@ -268,9 +268,9 @@ services:
      - --threads-batch
      - "${LLAMA_THREADS_BATCH:-6}"
      - --batch-size
-      - "64"
+      - "${LARGE_BATCH_SIZE:-2048}"
      - --ubatch-size
-      - "32"
+      - "${LARGE_UBATCH_SIZE:-128}"
      - --parallel
      - "1"
      - --jinja
@@ -345,9 +345,9 @@ services:
      - --threads-batch
      - "${LLAMA_THREADS_BATCH:-6}"
      - --batch-size
-      - "64"
+      - "${ULTRA_BATCH_SIZE:-2048}"
      - --ubatch-size
-      - "32"
+      - "${ULTRA_UBATCH_SIZE:-128}"
      - --parallel
      - "1"
      - --jinja
@@ -426,9 +426,9 @@ services:
      - --threads-batch
      - "${LLAMA_THREADS_BATCH:-6}"
      - --batch-size
-      - "64"
+      - "${UNCENSORED_BATCH_SIZE:-2048}"
      - --ubatch-size
-      - "32"
+      - "${UNCENSORED_UBATCH_SIZE:-128}"
      - --parallel
      - "1"
      - --jinja