Increase llama prompt cache to 24 GiB

2026-08-30 22:33:37 +02:00
parent a41d6dc28c
commit 0fa9a34886
3 changed files with 10 additions and 8 deletions
@@ -52,7 +52,8 @@ EXPERIMENTAL_GPU_DEVICES=0
 LLAMA_THREADS=6
 LLAMA_THREADS_BATCH=6
 FAST_PARALLEL_SLOTS=1
-MEDIUM_PARALLEL_SLOTS=2
+LLAMA_CACHE_RAM_MIB=24576
+MEDIUM_PARALLEL_SLOTS=1
 LARGE_PARALLEL_SLOTS=1
 ULTRA_PARALLEL_SLOTS=1
 UNCENSORED_PARALLEL_SLOTS=1