michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace vision hotswap with native multimodal profiles

Browse Source
This commit is contained in:
Mikei386
2026-08-20 14:24:56 +02:00
parent 8a45a0d851
commit cb07779f5a
19 changed files with 169 additions and 719 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/llama/README.md
+1 -1
View File
@@ -16,7 +16,7 @@ nach Standardbenchmark, Tool-Calling-Test und Kontexttest übernommen.
- Fast und Long: Qwen3.8-27B IQ4-MIX mit MTP2
- Medium: Qwen3.8-27B IQ4_XS Pure ohne MTP
- Vision-Hotswap: Qwen3.8-27B Q3_K_M plus BF16-mmproj
- Alle Profile: integrierte Vision mit demselben BF16-mmproj im System-RAM
Die Dateien selbst sind nicht Bestandteil des Repositories. Pfade und Hashes
werden im lokalen Modellmanifest verwaltet.
platform/models/manifest.example.yaml
+1 -7
View File
@@ -12,14 +12,8 @@ models:
file: qwen3.8-27b-IQ4_XS-pure.gguf
target: /opt/mike-ai/models/qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf
sha256: "REPLACE_AFTER_VERIFICATION"
qwen_vision:
role: temporary-vision-model
source: "REPLACE_WITH_MODEL_REPOSITORY"
file: Qwen3.8-27B-Q3_K_M.gguf
target: /opt/mike-ai/models/qwen3.8-27b/Qwen3.8-27B-Q3_K_M.gguf
sha256: "REPLACE_AFTER_VERIFICATION"
qwen_vision_projector:
role: vision-projector
role: integrated-vision-projector-for-all-qwen-profiles
source: "REPLACE_WITH_MODEL_REPOSITORY"
file: mmproj-BF16.gguf
target: /opt/mike-ai/models/qwen3.8-27b-nvfp4/mmproj-BF16.gguf
platform/profiles/profile-long.conf
+2 -2
View File
@@ -1,6 +1,6 @@
[Unit]
Description=Local AI llama.cpp - Qwen Long 128K MTP2 FFN12 CPU
Description=Local AI llama.cpp - Qwen Long 128K MTP2 FFN12 CPU with CPU Vision
[Service]
ExecStart=
ExecStart=/opt/mike-ai/llama.cpp/build/bin/llama-server --model /opt/mike-ai/models/qwen3.8-27b-iq4-mix/Qwen3.8-27B-IQ4-MIX.gguf --alias qwen38-27b-iq4mix-128k-mtp2-ffn12 --ctx-size 131072 --flash-attn on --cache-type-k q4_0 --cache-type-v q4_0 --threads 6 --threads-batch 6 --batch-size 64 --ubatch-size 32 --parallel 1 --jinja --host 127.0.0.1 --port 8080 --metrics --fit off --n-gpu-layers all --override-tensor blk.([0-9]|1[0-1]).ffn_.*=CPU --no-mmap --temperature 0.2 --top-p 0.8 --top-k 20 --mcp-servers-config /etc/mike-ai/mcp-servers.json --device CUDA0 --split-mode none --spec-type draft-mtp --spec-draft-n-max 2 --spec-draft-type-k q4_0 --spec-draft-type-v q4_0
ExecStart=/opt/mike-ai/llama.cpp/build/bin/llama-server --model /opt/mike-ai/models/qwen3.8-27b-iq4-mix/Qwen3.8-27B-IQ4-MIX.gguf --mmproj /opt/mike-ai/models/qwen3.8-27b-nvfp4/mmproj-BF16.gguf --no-mmproj-offload --alias qwen38-27b-iq4mix-128k-mtp2-ffn12 --ctx-size 131072 --flash-attn on --cache-type-k q4_0 --cache-type-v q4_0 --threads 6 --threads-batch 6 --batch-size 64 --ubatch-size 32 --parallel 1 --jinja --host 127.0.0.1 --port 8080 --metrics --fit off --n-gpu-layers all --override-tensor blk.([0-9]|1[0-1]).ffn_.*=CPU --no-mmap --temperature 0.2 --top-p 0.8 --top-k 20 --mcp-servers-config /etc/mike-ai/mcp-servers.json --device CUDA0 --split-mode none --spec-type draft-mtp --spec-draft-n-max 2 --spec-draft-type-k q4_0 --spec-draft-type-v q4_0
platform/profiles/profile-medium.conf
+2 -2
View File
@@ -1,6 +1,6 @@
[Unit]
Description=Local AI llama.cpp - Qwen Medium 92K IQ4_XS Pure
Description=Local AI llama.cpp - Qwen Medium 92K IQ4_XS Pure with CPU Vision
[Service]
ExecStart=
ExecStart=/opt/mike-ai/llama.cpp/build/bin/llama-server --model /opt/mike-ai/models/qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf --alias qwen38-27b-iq4xs-pure-92k --ctx-size 94208 --flash-attn on --cache-type-k q4_0 --cache-type-v q4_0 --threads 6 --threads-batch 6 --batch-size 64 --ubatch-size 32 --parallel 1 --jinja --reasoning auto --host 127.0.0.1 --port 8080 --metrics --fit off --n-gpu-layers all --no-mmap --temperature 0.2 --top-p 0.8 --top-k 20 --mcp-servers-config /etc/mike-ai/mcp-servers.json --device CUDA0 --split-mode none
ExecStart=/opt/mike-ai/llama.cpp/build/bin/llama-server --model /opt/mike-ai/models/qwen3.8-27b-iq4-xs-pure/qwen3.8-27b-IQ4_XS-pure.gguf --mmproj /opt/mike-ai/models/qwen3.8-27b-nvfp4/mmproj-BF16.gguf --no-mmproj-offload --alias qwen38-27b-iq4xs-pure-92k --ctx-size 94208 --flash-attn on --cache-type-k q4_0 --cache-type-v q4_0 --threads 6 --threads-batch 6 --batch-size 64 --ubatch-size 32 --parallel 1 --jinja --reasoning auto --host 127.0.0.1 --port 8080 --metrics --fit off --n-gpu-layers all --no-mmap --temperature 0.2 --top-p 0.8 --top-k 20 --mcp-servers-config /etc/mike-ai/mcp-servers.json --device CUDA0 --split-mode none