Record bounded Medium GPU split and MTP comparison

2026-09-20 22:42:09 +02:00
parent 3cbcf41fab
commit 635b3c4ba9
32 changed files with 5107 additions and 0 deletions
@@ -0,0 +1,75 @@
+[
+  "--model",
+  "/models/qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf",
+  "--mmproj",
+  "/models/qwen/mmproj-BF16.gguf",
+  "--mmproj-offload",
+  "--mmproj-device",
+  "CUDA1",
+  "--alias",
+  "qwen-medium",
+  "--ctx-size",
+  "160000",
+  "--flash-attn",
+  "on",
+  "--cache-type-k",
+  "q4_0",
+  "--cache-type-v",
+  "q4_0",
+  "--cache-prompt",
+  "--cache-ram",
+  "32768",
+  "--threads",
+  "6",
+  "--threads-batch",
+  "6",
+  "--batch-size",
+  "2048",
+  "--ubatch-size",
+  "256",
+  "--parallel",
+  "2",
+  "--kv-unified",
+  "--jinja",
+  "--reasoning",
+  "auto",
+  "--reasoning-preserve",
+  "--host",
+  "127.0.0.1",
+  "--port",
+  "5005",
+  "--metrics",
+  "--fit",
+  "off",
+  "--n-gpu-layers",
+  "all",
+  "--load-mode",
+  "none",
+  "--no-ui",
+  "--temperature",
+  "1.0",
+  "--top-p",
+  "0.95",
+  "--top-k",
+  "20",
+  "--device",
+  "CUDA0,CUDA1",
+  "--main-gpu",
+  "0",
+  "--split-mode",
+  "layer",
+  "--tensor-split",
+  "85,15",
+  "--spec-type",
+  "draft-mtp",
+  "--spec-draft-n-max",
+  "2",
+  "--spec-draft-type-k",
+  "f16",
+  "--spec-draft-type-v",
+  "f16",
+  "--spec-draft-p-min",
+  "0.05",
+  "--verbosity",
+  "3"
+]