michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Increase llama prompt cache to 24 GiB

Browse Source
This commit is contained in:
Mikei386
2026-08-30 22:33:37 +02:00
parent a41d6dc28c
commit 0fa9a34886
3 changed files with 10 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+2 -1
View File
@@ -52,7 +52,8 @@ EXPERIMENTAL_GPU_DEVICES=0
LLAMA_THREADS=6
LLAMA_THREADS_BATCH=6
FAST_PARALLEL_SLOTS=1
MEDIUM_PARALLEL_SLOTS=2
LLAMA_CACHE_RAM_MIB=24576
MEDIUM_PARALLEL_SLOTS=1
LARGE_PARALLEL_SLOTS=1
ULTRA_PARALLEL_SLOTS=1
UNCENSORED_PARALLEL_SLOTS=1
compose.yaml
+6 -6
View File
@@ -99,7 +99,7 @@ services:
# disabled until the current upstream restore regressions are fixed.
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -176,7 +176,7 @@ services:
- q4_0
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -264,7 +264,7 @@ services:
- q4_0
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -342,7 +342,7 @@ services:
- --cache-reuse
- "${LLAMA_CACHE_REUSE:-256}"
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -424,7 +424,7 @@ services:
- q4_0
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -501,7 +501,7 @@ services:
- --cache-reuse
- "${LLAMA_CACHE_REUSE:-256}"
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- --parallel
- "${EXPERIMENTAL_PARALLEL_SLOTS:-1}"
- --kv-unified
config/install.env.example
+2 -1
View File
@@ -100,7 +100,8 @@ EXPERIMENTAL_CONTEXT=76800
LLAMA_THREADS=6
LLAMA_THREADS_BATCH=6
FAST_PARALLEL_SLOTS=1
MEDIUM_PARALLEL_SLOTS=2
LLAMA_CACHE_RAM_MIB=24576
MEDIUM_PARALLEL_SLOTS=1
LARGE_PARALLEL_SLOTS=1
ULTRA_PARALLEL_SLOTS=1
UNCENSORED_PARALLEL_SLOTS=1