Enable benchmarked two-slot medium profile

2026-08-30 20:54:22 +02:00
parent db3719af7a
commit a41d6dc28c
7 changed files with 60 additions and 21 deletions
@@ -109,7 +109,8 @@ services:
      - --ubatch-size
      - "${FAST_UBATCH_SIZE:-32}"
      - --parallel
-      - "1"
+      - "${FAST_PARALLEL_SLOTS:-1}"
+      - --kv-unified
      - --jinja
      - --reasoning
      - auto
@@ -185,7 +186,8 @@ services:
      - --ubatch-size
      - "${MEDIUM_UBATCH_SIZE:-128}"
      - --parallel
-      - "1"
+      - "${MEDIUM_PARALLEL_SLOTS:-2}"
+      - --kv-unified
      - --jinja
      - --reasoning
      - auto
@@ -221,7 +223,7 @@ services:
      - --split-mode
      - layer
      - --tensor-split
-      - "${MEDIUM_TENSOR_SPLIT:-90,10}"
+      - "${MEDIUM_TENSOR_SPLIT:-85,15}"
      - --spec-type
      - draft-mtp
      - --spec-draft-n-max
@@ -272,7 +274,8 @@ services:
      - --ubatch-size
      - "${LARGE_UBATCH_SIZE:-128}"
      - --parallel
-      - "1"
+      - "${LARGE_PARALLEL_SLOTS:-1}"
+      - --kv-unified
      - --jinja
      - --reasoning
      - auto
@@ -349,7 +352,8 @@ services:
      - --ubatch-size
      - "${ULTRA_UBATCH_SIZE:-128}"
      - --parallel
-      - "1"
+      - "${ULTRA_PARALLEL_SLOTS:-1}"
+      - --kv-unified
      - --jinja
      - --reasoning
      - auto
@@ -430,7 +434,8 @@ services:
      - --ubatch-size
      - "${UNCENSORED_UBATCH_SIZE:-128}"
      - --parallel
-      - "1"
+      - "${UNCENSORED_PARALLEL_SLOTS:-1}"
+      - --kv-unified
      - --jinja
      - --reasoning
      - auto
@@ -498,7 +503,8 @@ services:
      - --cache-ram
      - "${LLAMA_CACHE_RAM_MIB:-8192}"
      - --parallel
-      - "1"
+      - "${EXPERIMENTAL_PARALLEL_SLOTS:-1}"
+      - --kv-unified
      - --jinja
      - --reasoning
      - auto