From 9c10b0ab88cfae620df282ebf2dbac8ff308067a Mon Sep 17 00:00:00 2001
From: Mikei386 <44135113+Mikei386@users.noreply.github.com>
Date: Sun, 30 Aug 2026 18:39:17 +0200
Subject: [PATCH] Tune Qwen medium prefill batching

---
 .env.example               | 2 ++
 compose.yaml               | 4 ++--
 config/install.env.example | 2 ++
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.env.example b/.env.example
index 6664b35..7d9831b 100644
--- a/.env.example
+++ b/.env.example
@@ -23,6 +23,8 @@ VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf
 
 FAST_CONTEXT=76800
 MEDIUM_CONTEXT=160000
+MEDIUM_BATCH_SIZE=2048
+MEDIUM_UBATCH_SIZE=128
 LARGE_CONTEXT=192000
 ULTRA_CONTEXT=262144
 UNCENSORED_CONTEXT=80000
diff --git a/compose.yaml b/compose.yaml
index c4308df..b8a9385 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -181,9 +181,9 @@ services:
       - --threads-batch
       - "${LLAMA_THREADS_BATCH:-6}"
       - --batch-size
-      - "64"
+      - "${MEDIUM_BATCH_SIZE:-2048}"
       - --ubatch-size
-      - "32"
+      - "${MEDIUM_UBATCH_SIZE:-128}"
       - --parallel
       - "1"
       - --jinja
diff --git a/config/install.env.example b/config/install.env.example
index 5422a90..07540cf 100644
--- a/config/install.env.example
+++ b/config/install.env.example
@@ -77,6 +77,8 @@ VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c239
 
 FAST_CONTEXT=76800
 MEDIUM_CONTEXT=160000
+MEDIUM_BATCH_SIZE=2048
+MEDIUM_UBATCH_SIZE=128
 MEDIUM_TENSOR_SPLIT=90,10
 LARGE_CONTEXT=192000
 LARGE_TENSOR_SPLIT=86,14