Harden and demote experimental Hermes compressor

This commit is contained in:
Mikei386
2026-08-26 01:40:38 +02:00
parent 99285fe81a
commit aaadc57c07
7 changed files with 52 additions and 40 deletions
+16 -1
View File
@@ -97,7 +97,7 @@ services:
cap_drop: [ALL]
command:
- --model
- "/models/${COMPRESSION_MODEL_FILE:-qwen3.5-2b-compression/Qwen_Qwen3.5-2B-Q4_K_M.gguf}"
- "/models/${COMPRESSION_MODEL_FILE:-qwen3.5-4b-compression/Qwen_Qwen3.5-4B-Q4_K_M.gguf}"
- --alias
- qwen-compression
- --ctx-size
@@ -117,6 +117,21 @@ services:
- --port
- "8099"
- --metrics
# Official Qwen3.5 non-thinking sampler. The presence penalty is
# essential here: without it one technical YAML benchmark repeated the
# summary structure for more than 40K tokens.
- --temperature
- "1.0"
- --top-p
- "1.0"
- --top-k
- "20"
- --presence-penalty
- "2.0"
# Hermes' own summary ceiling is 10K. Keep a little completion margin,
# but never allow an accidental repetition loop to fill the 65K slot.
- --n-predict
- "12000"
- --n-gpu-layers
- all
- --device