Add dedicated Hermes compression model

2026-08-26 00:45:32 +02:00
parent e872829b11
commit 43d9903d13
8 changed files with 120 additions and 3 deletions
@@ -12,6 +12,11 @@ XTTS_CACHE_DIR=/data/models/xtts-v2-cache
 XTTS_GPU_DEVICE=GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
 AI_DNS=192.168.1.1

+# Dedicated Hermes context compressor on the RTX 3060.
+COMPRESSION_MODEL_FILE=qwen3.5-2b-compression/Qwen_Qwen3.5-2B-Q4_K_M.gguf
+COMPRESSION_CONTEXT=65536
+COMPRESSION_GPU_DEVICE=GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
+
 FAST_MODEL_FILE=qwen-mix/Qwen3.8-27B-IQ4-MIX.gguf
 MEDIUM_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf
 LARGE_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf