diff --git a/.env.example b/.env.example index f8418bc..d6c9b28 100644 --- a/.env.example +++ b/.env.example @@ -52,7 +52,8 @@ EXPERIMENTAL_GPU_DEVICES=0 LLAMA_THREADS=6 LLAMA_THREADS_BATCH=6 FAST_PARALLEL_SLOTS=1 -MEDIUM_PARALLEL_SLOTS=2 +LLAMA_CACHE_RAM_MIB=24576 +MEDIUM_PARALLEL_SLOTS=1 LARGE_PARALLEL_SLOTS=1 ULTRA_PARALLEL_SLOTS=1 UNCENSORED_PARALLEL_SLOTS=1 diff --git a/compose.yaml b/compose.yaml index 55113c0..b633fb4 100644 --- a/compose.yaml +++ b/compose.yaml @@ -99,7 +99,7 @@ services: # disabled until the current upstream restore regressions are fixed. - --cache-prompt - --cache-ram - - "${LLAMA_CACHE_RAM_MIB:-8192}" + - "${LLAMA_CACHE_RAM_MIB:-24576}" - --threads - "${LLAMA_THREADS:-6}" - --threads-batch @@ -176,7 +176,7 @@ services: - q4_0 - --cache-prompt - --cache-ram - - "${LLAMA_CACHE_RAM_MIB:-8192}" + - "${LLAMA_CACHE_RAM_MIB:-24576}" - --threads - "${LLAMA_THREADS:-6}" - --threads-batch @@ -264,7 +264,7 @@ services: - q4_0 - --cache-prompt - --cache-ram - - "${LLAMA_CACHE_RAM_MIB:-8192}" + - "${LLAMA_CACHE_RAM_MIB:-24576}" - --threads - "${LLAMA_THREADS:-6}" - --threads-batch @@ -342,7 +342,7 @@ services: - --cache-reuse - "${LLAMA_CACHE_REUSE:-256}" - --cache-ram - - "${LLAMA_CACHE_RAM_MIB:-8192}" + - "${LLAMA_CACHE_RAM_MIB:-24576}" - --threads - "${LLAMA_THREADS:-6}" - --threads-batch @@ -424,7 +424,7 @@ services: - q4_0 - --cache-prompt - --cache-ram - - "${LLAMA_CACHE_RAM_MIB:-8192}" + - "${LLAMA_CACHE_RAM_MIB:-24576}" - --threads - "${LLAMA_THREADS:-6}" - --threads-batch @@ -501,7 +501,7 @@ services: - --cache-reuse - "${LLAMA_CACHE_REUSE:-256}" - --cache-ram - - "${LLAMA_CACHE_RAM_MIB:-8192}" + - "${LLAMA_CACHE_RAM_MIB:-24576}" - --parallel - "${EXPERIMENTAL_PARALLEL_SLOTS:-1}" - --kv-unified diff --git a/config/install.env.example b/config/install.env.example index e12c4e6..da3221f 100644 --- a/config/install.env.example +++ b/config/install.env.example @@ -100,7 +100,8 @@ EXPERIMENTAL_CONTEXT=76800 LLAMA_THREADS=6 LLAMA_THREADS_BATCH=6 FAST_PARALLEL_SLOTS=1 -MEDIUM_PARALLEL_SLOTS=2 +LLAMA_CACHE_RAM_MIB=24576 +MEDIUM_PARALLEL_SLOTS=1 LARGE_PARALLEL_SLOTS=1 ULTRA_PARALLEL_SLOTS=1 UNCENSORED_PARALLEL_SLOTS=1