michael/AI-Profile-Router
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use embedded MTP tensor for IQ4-MIX profiles

Browse Source
This commit is contained in:
Mikei386 committed 2026-08-21 13:18:45 +02:00
1 parent 057920b941
commit 42e2aad3a6
4 files changed
+3 -12
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.env.example
-1
View File
@@ -12,7 +12,6 @@ MEDIUM_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf
LONG_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf LONG_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf
EXPERIMENTAL_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf EXPERIMENTAL_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf
VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf
MTP_MODEL_FILE=qwen/mtp-Qwen3.8-27B-Q4_0.gguf
FAST_CONTEXT=76800 FAST_CONTEXT=76800
MEDIUM_CONTEXT=94208 MEDIUM_CONTEXT=94208
compose.yaml
-4
View File
@@ -87,8 +87,6 @@ services:
- none - none
- --spec-type - --spec-type
- draft-mtp - draft-mtp
- --model-draft
- "/models/${MTP_MODEL_FILE:?MTP_MODEL_FILE is required}"
- --spec-draft-n-max - --spec-draft-n-max
- "2" - "2"
- --spec-draft-type-k - --spec-draft-type-k
@@ -217,8 +215,6 @@ services:
- none - none
- --spec-type - --spec-type
- draft-mtp - draft-mtp
- --model-draft
- "/models/${MTP_MODEL_FILE:?MTP_MODEL_FILE is required}"
- --spec-draft-n-max - --spec-draft-n-max
- "2" - "2"
- --spec-draft-type-k - --spec-draft-type-k
config/install.env.example
+2 -3
View File
@@ -43,9 +43,8 @@ EXPERIMENTAL_MODEL_SHA256=40fac4050e940397dbf13087afd50f4734a11805bf9d65ef8ddd74
VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf
VISION_PROJECTOR_URL=https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/mmproj-BF16.gguf VISION_PROJECTOR_URL=https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/mmproj-BF16.gguf
VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c2395f5c2d53 VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c2395f5c2d53
MTP_MODEL_FILE=qwen/mtp-Qwen3.8-27B-Q4_0.gguf # FAST/LONG use the MTP tensor embedded in the IQ4-MIX GGUF. A separate
MTP_MODEL_URL=https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/MTP/mtp-Qwen3.8-27B-Q4_0.gguf # draft-model artifact is neither downloaded nor passed to llama-server.
MTP_MODEL_SHA256=50d9ce5a6da381bbcfb31061cf73df94a90e6faf8efeddee379a9cb8f1501c6e
FAST_CONTEXT=76800 FAST_CONTEXT=76800
MEDIUM_CONTEXT=94208 MEDIUM_CONTEXT=94208
install.sh
+1 -4
View File
@@ -42,8 +42,7 @@ required=(AI_HOSTNAME ADMIN_USER AI_BIND_ADDRESS MODEL_DIR FAST_MODEL_FILE
FAST_MODEL_URL FAST_MODEL_SHA256 MEDIUM_MODEL_FILE MEDIUM_MODEL_URL FAST_MODEL_URL FAST_MODEL_SHA256 MEDIUM_MODEL_FILE MEDIUM_MODEL_URL
MEDIUM_MODEL_SHA256 LONG_MODEL_FILE LONG_MODEL_URL LONG_MODEL_SHA256 MEDIUM_MODEL_SHA256 LONG_MODEL_FILE LONG_MODEL_URL LONG_MODEL_SHA256
EXPERIMENTAL_MODEL_FILE EXPERIMENTAL_MODEL_URL EXPERIMENTAL_MODEL_SHA256 EXPERIMENTAL_MODEL_FILE EXPERIMENTAL_MODEL_URL EXPERIMENTAL_MODEL_SHA256
VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256 VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256)
MTP_MODEL_FILE MTP_MODEL_URL MTP_MODEL_SHA256)
for name in "${required[@]}"; do for name in "${required[@]}"; do
[[ -n "${!name:-}" ]] || die "Pflichtwert $name fehlt." [[ -n "${!name:-}" ]] || die "Pflichtwert $name fehlt."
done done
@@ -202,7 +201,6 @@ MEDIUM_MODEL_FILE=$MEDIUM_MODEL_FILE
LONG_MODEL_FILE=$LONG_MODEL_FILE LONG_MODEL_FILE=$LONG_MODEL_FILE
EXPERIMENTAL_MODEL_FILE=$EXPERIMENTAL_MODEL_FILE EXPERIMENTAL_MODEL_FILE=$EXPERIMENTAL_MODEL_FILE
VISION_PROJECTOR_FILE=$VISION_PROJECTOR_FILE VISION_PROJECTOR_FILE=$VISION_PROJECTOR_FILE
MTP_MODEL_FILE=$MTP_MODEL_FILE
FAST_CONTEXT=${FAST_CONTEXT:-76800} FAST_CONTEXT=${FAST_CONTEXT:-76800}
MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-94208} MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-94208}
LONG_CONTEXT=${LONG_CONTEXT:-131072} LONG_CONTEXT=${LONG_CONTEXT:-131072}
@@ -245,7 +243,6 @@ $MEDIUM_MODEL_FILE|$MEDIUM_MODEL_URL|$MEDIUM_MODEL_SHA256
$LONG_MODEL_FILE|$LONG_MODEL_URL|$LONG_MODEL_SHA256 $LONG_MODEL_FILE|$LONG_MODEL_URL|$LONG_MODEL_SHA256
$EXPERIMENTAL_MODEL_FILE|$EXPERIMENTAL_MODEL_URL|$EXPERIMENTAL_MODEL_SHA256 $EXPERIMENTAL_MODEL_FILE|$EXPERIMENTAL_MODEL_URL|$EXPERIMENTAL_MODEL_SHA256
$VISION_PROJECTOR_FILE|$VISION_PROJECTOR_URL|$VISION_PROJECTOR_SHA256 $VISION_PROJECTOR_FILE|$VISION_PROJECTOR_URL|$VISION_PROJECTOR_SHA256
$MTP_MODEL_FILE|$MTP_MODEL_URL|$MTP_MODEL_SHA256
EOF EOF
} }