Record bounded Medium GPU split and MTP comparison

2026-09-20 22:42:09 +02:00
parent 3cbcf41fab
commit 635b3c4ba9
32 changed files with 5107 additions and 0 deletions
@@ -0,0 +1,76 @@
+{
+  "image": "sha256:5e3c12c145b8045e5731b44b6b97033f24b327ae3d4a3fa85ecdd159cc844907",
+  "args": [
+    "--model",
+    "/models/qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf",
+    "--mmproj",
+    "/models/qwen/mmproj-BF16.gguf",
+    "--mmproj-offload",
+    "--mmproj-device",
+    "CUDA1",
+    "--alias",
+    "qwen-medium",
+    "--ctx-size",
+    "160000",
+    "--flash-attn",
+    "on",
+    "--cache-type-k",
+    "q4_0",
+    "--cache-type-v",
+    "q4_0",
+    "--cache-prompt",
+    "--cache-ram",
+    "32768",
+    "--threads",
+    "6",
+    "--threads-batch",
+    "6",
+    "--batch-size",
+    "2048",
+    "--ubatch-size",
+    "512",
+    "--parallel",
+    "2",
+    "--kv-unified",
+    "--jinja",
+    "--reasoning",
+    "auto",
+    "--reasoning-preserve",
+    "--host",
+    "0.0.0.0",
+    "--port",
+    "8080",
+    "--metrics",
+    "--fit",
+    "off",
+    "--n-gpu-layers",
+    "all",
+    "--load-mode",
+    "none",
+    "--no-ui",
+    "--temperature",
+    "1.0",
+    "--top-p",
+    "0.95",
+    "--top-k",
+    "20",
+    "--device",
+    "CUDA0,CUDA1",
+    "--main-gpu",
+    "0",
+    "--split-mode",
+    "layer",
+    "--tensor-split",
+    "85,15",
+    "--spec-type",
+    "draft-mtp",
+    "--spec-draft-n-max",
+    "3",
+    "--spec-draft-type-k",
+    "f16",
+    "--spec-draft-type-v",
+    "f16",
+    "--spec-draft-p-min",
+    "0.05"
+  ]
+}