Increase llama prompt cache to 32 GiB

2026-09-03 10:03:43 +02:00
parent 4153e535d3
commit fb0cb40bed
4 changed files with 8 additions and 8 deletions
@@ -358,7 +358,7 @@ IMAGE_GPU_DEVICES=${IMAGE_GPU_DEVICES:-${TEXT_GPU_DEVICES:-0}}
 FLUX_MODEL_DIR=${FLUX_MODEL_DIR:-/data/models/FLUX.2-klein-4B}
 LLAMA_THREADS=${LLAMA_THREADS:-6}
 LLAMA_THREADS_BATCH=${LLAMA_THREADS_BATCH:-6}
-LLAMA_CACHE_RAM_MIB=${LLAMA_CACHE_RAM_MIB:-24576}
+LLAMA_CACHE_RAM_MIB=${LLAMA_CACHE_RAM_MIB:-32768}
 DEFAULT_REASONING_EFFORT=${DEFAULT_REASONING_EFFORT:-off}
 EOF
  chmod 0600 $SECRETS_DIR/stack.env