From a067b79ed28c002e73125dad648ecbb7c0b44b5a Mon Sep 17 00:00:00 2001 From: Mikei386 <44135113+Mikei386@users.noreply.github.com> Date: Sun, 23 Aug 2026 13:16:17 +0200 Subject: [PATCH] Stabilize German XTTS speech output --- compose.yaml | 3 + docs/ARCHITECTURE.md | 9 +- docs/COMPONENTS.md | 2 +- docs/INSTALLATION.md | 8 +- .../docker/tts-gateway/test_tts_gateway.py | 64 ++++++ platform/docker/tts-gateway/tts_gateway.py | 195 +++++++++++++++++- 6 files changed, 264 insertions(+), 17 deletions(-) diff --git a/compose.yaml b/compose.yaml index 3eae357..47650dd 100644 --- a/compose.yaml +++ b/compose.yaml @@ -680,6 +680,9 @@ services: TTS_VOICE_ALIAS: alloy XTTS_SPEAKER: Annmarie Nele TTS_DEFAULT_LANGUAGE: de + # Mixed-language clip stitching caused long pauses and unintelligible + # transitions. Keep full sentences in one stable German voice. + TTS_CODE_SWITCH_ENABLED: "false" XTTS_QUEUE_TIMEOUT: "15" XTTS_TIMEOUT: "120" PIPER_TIMEOUT: "120" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e939272..2503636 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -105,9 +105,12 @@ Ultra bleibt für maximalen Kontext bewusst text-only. Open WebUI spricht ausschließlich den Router an. Dieser reicht TTS intern an das TTS-Gateway weiter. Das Gateway nutzt primär XTTS-v2 mit der Stimme -`Annmarie Nele` auf der RTX 3060. Deutsche Texte werden an bekannten -englischen IT-Begriffen segmentiert; reine englische Texte laufen vollständig -mit `language=en`. Da der offizielle XTTS-Streamingserver nur einen Auftrag +`Annmarie Nele` auf der RTX 3060. Gemischte deutsch-englische Antworten laufen +bewusst als vollständige deutsche Satzblöcke: Das vermeidet die langen Pausen, +Tonhöhensprünge und unverständlichen Übergänge, die beim Zusammensetzen vieler +kurzer Sprachsegmente entstanden. Vollständig englische Texte werden weiterhin +automatisch mit `language=en` gesprochen. Da der offizielle +XTTS-Streamingserver nur einen Auftrag gleichzeitig unterstützt, serialisiert das Gateway die Aufträge. Bei Fehler, Timeout oder belegter Queue übernimmt automatisch Piper auf der CPU. Kein TTS-Port wird veröffentlicht. Der äußere Kompatibilitätsname bleibt bewusst diff --git a/docs/COMPONENTS.md b/docs/COMPONENTS.md index 60297e0..d0bf325 100644 --- a/docs/COMPONENTS.md +++ b/docs/COMPONENTS.md @@ -13,7 +13,7 @@ | Unraid-MCP | lokales `runraid`-Binary | eigener optionaler Container | optional | | Whisper | ggml-org/whisper.cpp | Service im Router-Deploy | optional | | XTTS-v2 | Coqui, offizielles CUDA-12.1-Image per Digest | RTX-3060-Container, Stimme `Annmarie Nele`, CPML | Kern | -| TTS-Gateway | `platform/docker/tts-gateway/` | interne Queue, Deutsch/Englisch-Segmentierung und Piper-Fallback | Kern | +| TTS-Gateway | `platform/docker/tts-gateway/` | interne Queue, stabile deutsche Satzblöcke, automatische Erkennung rein englischer Texte und Piper-Fallback | Kern | | Piper TTS | Open Home Foundation, `piper-tts` 1.6.0 | interner CPU-Fallback, Stimme `de_DE-thorsten-high` | Kern | | FLUX.2 klein | Black Forest Labs | Worker und Modellmanifest | optional | | LLama-GUI | separates Upstream-Projekt | nur Betriebsrolle dokumentiert | optional | diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md index e920846..889d2de 100644 --- a/docs/INSTALLATION.md +++ b/docs/INSTALLATION.md @@ -88,8 +88,12 @@ intern an das TTS-Gateway weiter. Primär spricht XTTS-v2 mit `Annmarie Nele` auf der RTX 3060; bei Fehlern oder Queue-Timeout übernimmt Piper auf der CPU. Der Port 8085 wird nicht am Host veröffentlicht. Ein Restore setzt zusätzlich die vier persistenten Audiofelder gezielt neu, damit -alte Werte wie `tts-1` oder `coral` die Compose-Vorgaben nicht überstimmen. Ein -Ende-zu-Ende-Test ohne Ausgabe des API-Schlüssels: +alte Werte wie `tts-1` oder `coral` die Compose-Vorgaben nicht überstimmen. +`TTS_CODE_SWITCH_ENABLED=false` hält gemischte Antworten als zusammenhängende +deutsche Satzblöcke. Einzelne englische Fachbegriffe werden damit zwar deutsch +ausgesprochen, die Ausgabe bleibt jedoch flüssig und verständlich. Reine +englische Texte erkennt das Gateway weiterhin automatisch. Ein Ende-zu-Ende-Test +ohne Ausgabe des API-Schlüssels: ```bash set -a; source /etc/mike-ai/stack.env; set +a diff --git a/platform/docker/tts-gateway/test_tts_gateway.py b/platform/docker/tts-gateway/test_tts_gateway.py index 71e8dc5..15fe35c 100644 --- a/platform/docker/tts-gateway/test_tts_gateway.py +++ b/platform/docker/tts-gateway/test_tts_gateway.py @@ -1,6 +1,7 @@ import importlib.util import pathlib import unittest +from array import array MODULE_PATH = pathlib.Path(__file__).with_name("tts_gateway.py") @@ -10,6 +11,12 @@ SPEC.loader.exec_module(gateway) class LanguageSegmentationTests(unittest.TestCase): + def setUp(self): + self.original_code_switch = gateway.CODE_SWITCH_ENABLED + + def tearDown(self): + gateway.CODE_SWITCH_ENABLED = self.original_code_switch + def test_german_only(self): self.assertEqual( gateway.segment_languages("Guten Abend, wie warm ist es heute?"), @@ -21,6 +28,7 @@ class LanguageSegmentationTests(unittest.TestCase): self.assertEqual(gateway.segment_languages(text), [("en", text)]) def test_mixed_compounds(self): + gateway.CODE_SWITCH_ENABLED = True text = "Ich öffne das Unraid-Dashboard und prüfe die Docker-Container." self.assertEqual( gateway.segment_languages(text), @@ -33,6 +41,43 @@ class LanguageSegmentationTests(unittest.TestCase): ], ) + def test_operational_vocabulary_is_not_read_as_german(self): + gateway.CODE_SWITCH_ENABLED = True + text = ( + "31 von 60 Containern laufen, alle sind up und healthy. " + "Die Health-Checks zeigen keine Crashes oder Restart-Loops. " + "Immich meldet unsupported image format und premature end of JPEG." + ) + segments = gateway.segment_languages(text) + english = [part.lower() for language, part in segments if language == "en"] + self.assertIn("containern", english) + self.assertIn("up", english) + self.assertIn("healthy", english) + self.assertIn("health-checks", english) + self.assertIn("crashes", english) + self.assertIn("restart-loops", english) + self.assertIn("unsupported image format", english) + self.assertIn("premature end of jpeg", english) + prepared = gateway.prepare_segments(text) + self.assertTrue(all(any(char.isalnum() for char in part) for _, part in prepared)) + + def test_long_text_is_cleaned_and_chunked(self): + text = "✅ **Status**\n" + ("Ein langer deutscher Diagnosesatz. " * 30) + segments = gateway.prepare_segments(text) + self.assertTrue(segments) + self.assertTrue(all(len(part) <= gateway.CHUNK_CHARS for _, part in segments)) + self.assertNotIn("✅", "".join(part for _, part in segments)) + + def test_inflected_container_has_english_pronunciation(self): + gateway.CODE_SWITCH_ENABLED = True + prepared = gateway.prepare_segments("Ich prüfe die Containern.") + self.assertIn(("en", "containers."), prepared) + + def test_code_switch_is_disabled_by_default(self): + gateway.CODE_SWITCH_ENABLED = False + text = "Docker-Container laufen, die Health-Checks melden healthy." + self.assertEqual(gateway.segment_languages(text), [("de", text)]) + class FallbackTests(unittest.TestCase): def setUp(self): @@ -55,5 +100,24 @@ class FallbackTests(unittest.TestCase): ) +class AudioJoinTests(unittest.TestCase): + @staticmethod + def pcm(values): + samples = array("h", values) + return samples.tobytes() + + def test_trim_removes_long_silent_edges(self): + pcm = self.pcm(([0] * 1000) + ([500] * 100) + ([0] * 1000)) + trimmed = gateway._trim_pcm(pcm) + self.assertLess(len(trimmed), len(pcm)) + self.assertGreater(len(trimmed), 100 * 2) + + def test_join_inserts_pause_only_after_sentence(self): + spoken = self.pcm([500] * 1000) + inline = gateway._join_pcm([("Docker", spoken), ("Container", spoken)]) + sentence = gateway._join_pcm([("Fertig.", spoken), ("Weiter", spoken)]) + self.assertGreater(len(sentence), len(inline)) + + if __name__ == "__main__": unittest.main() diff --git a/platform/docker/tts-gateway/tts_gateway.py b/platform/docker/tts-gateway/tts_gateway.py index 05fc929..c42b115 100644 --- a/platform/docker/tts-gateway/tts_gateway.py +++ b/platform/docker/tts-gateway/tts_gateway.py @@ -14,9 +14,11 @@ import re import subprocess import threading import time +import unicodedata import urllib.error import urllib.request import wave +from array import array from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -28,13 +30,19 @@ PIPER_URL = os.getenv("PIPER_URL", "http://piper:8085").rstrip("/") VOICE_ALIAS = os.getenv("TTS_VOICE_ALIAS", "alloy") XTTS_SPEAKER = os.getenv("XTTS_SPEAKER", "Annmarie Nele") DEFAULT_LANGUAGE = os.getenv("TTS_DEFAULT_LANGUAGE", "de") +CODE_SWITCH_ENABLED = os.getenv("TTS_CODE_SWITCH_ENABLED", "false").lower() \ + in {"1", "true", "yes", "on"} MAX_TEXT_CHARS = int(os.getenv("TTS_MAX_TEXT_CHARS", "8000")) MAX_REQUEST_BYTES = int(os.getenv("TTS_MAX_REQUEST_BYTES", "65536")) MAX_AUDIO_BYTES = int(os.getenv("TTS_MAX_AUDIO_BYTES", str(64 * 1024 * 1024))) XTTS_TIMEOUT = float(os.getenv("XTTS_TIMEOUT", "120")) PIPER_TIMEOUT = float(os.getenv("PIPER_TIMEOUT", "120")) QUEUE_TIMEOUT = float(os.getenv("XTTS_QUEUE_TIMEOUT", "15")) -SILENCE_MS = int(os.getenv("XTTS_SEGMENT_SILENCE_MS", "20")) +CHUNK_CHARS = int(os.getenv("XTTS_CHUNK_CHARS", "220")) +TRIM_THRESHOLD = int(os.getenv("XTTS_TRIM_THRESHOLD", "90")) +TRIM_PADDING_MS = int(os.getenv("XTTS_TRIM_PADDING_MS", "18")) +CROSSFADE_MS = int(os.getenv("XTTS_CROSSFADE_MS", "8")) +SENTENCE_PAUSE_MS = int(os.getenv("XTTS_SENTENCE_PAUSE_MS", "65")) SYNTHESIS_LOCK = threading.Lock() STATE_LOCK = threading.Lock() @@ -50,16 +58,27 @@ STATE = { # Prefer full compounds to isolated terms. This keeps switches infrequent and # avoids making mixed-language speech sound like a sequence of separate clips. ENGLISH_TERMS = ( + "unsupported image format", "premature end of JPEG", "incomplete scan", + "Docker Containers", "Docker-Containers", "Docker Containern", + "Docker-Containern", "Health Checks", "Health-Checks", + "Restart Loops", "Restart-Loops", "False Positive", "Delivery Errors", + "DeliveryErrors", "Ack Problem", "Ack-Problem", "I/O timeout", + "Parity Check", "Parity-Check", "Disk disabled", "Disk invalid", "Home Assistant", "Open WebUI", "OpenWebUI", "Unraid Dashboard", "Unraid-Dashboard", "Docker Container", "Docker-Container", "Server Log", "Server-Log", "GitHub Repository", "GitHub Repo", "WireGuard Tunnel", "Cron Job", "Cronjob", "Home Server", "API Key", "Tool Calling", "Context Window", "Prompt Injection", - "Unraid", "Docker", "Container", "Dashboard", "Server", "Log", + "Unraid", "Docker", "Containern", "Containers", "Container", + "Crashes", "healthy", "disabled", "invalid", "Dashboard", "Server", + "Logs", "Log", "Matches", "up", "OpenAI", "GitHub", "WireGuard", "Linux", "Debian", "Frontend", "Backend", "Router", "Browser", "Web", "Token", "Prompt", "Context", "Model", "Image", "Tool", "Workflow", "Benchmark", "Streaming", "SSH", "MCP", "API", "CPU", "GPU", "VRAM", "RAM", "HTTP", "HTTPS", + "HomeAssistant", "ESPHome", "iGotify", "Immich", "Vaultwarden", + "UniFi", "UptimeKuma", "go2rtc", "Zigbee", "SONOFF", "eWeLink", + "RTSP", "JPEG", "NVMe", "GiB", ) TERM_PATTERN = re.compile( r"(? tuple[bytes, str]: @@ -124,6 +163,8 @@ def segment_languages(text: str) -> list[tuple[str, str]]: return [("en", text)] if DEFAULT_LANGUAGE != "de": return [(DEFAULT_LANGUAGE, text)] + if not CODE_SWITCH_ENABLED: + return [("de", text)] segments: list[tuple[str, str]] = [] cursor = 0 @@ -149,6 +190,79 @@ def segment_languages(text: str) -> list[tuple[str, str]]: return merged +def clean_for_speech(text: str) -> str: + """Remove visual markup that makes long TTS output unstable or noisy.""" + text = re.sub(r"```.*?```", " Code block. ", text, flags=re.DOTALL) + text = re.sub(r"`([^`]+)`", r"\1", text) + text = re.sub(r"!\[([^]]*)\]\([^)]+\)", r"\1", text) + text = re.sub(r"\[([^]]+)\]\([^)]+\)", r"\1", text) + text = re.sub(r"(?m)^\s{0,3}#{1,6}\s*", "", text) + text = re.sub(r"(?m)^\s*[-*+]\s+", "", text) + text = text.replace("→", ". ").replace("←", ". ") + text = text.replace("–", " - ").replace("—", " - ") + text = "".join( + char for char in text + if unicodedata.category(char) not in {"So", "Cs"} + ) + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\s*\n+\s*", ". ", text) + text = re.sub(r"(?:\.\s*){2,}", ". ", text) + return text.strip() + + +def _split_chunk(text: str, limit: int = CHUNK_CHARS) -> list[str]: + """Split at natural pauses and keep every XTTS request comfortably short.""" + text = text.strip() + if not text: + return [] + if len(text) <= limit: + return [text] + + pieces = re.split(r"(?<=[.!?;:])\s+|\s+(?=\d+[.)]\s)", text) + chunks: list[str] = [] + current = "" + for piece in pieces: + piece = piece.strip() + if not piece: + continue + if len(piece) > limit: + words = piece.split() + for word in words: + candidate = f"{current} {word}".strip() + if current and len(candidate) > limit: + chunks.append(current) + current = word + else: + current = candidate + continue + candidate = f"{current} {piece}".strip() + if current and len(candidate) > limit: + chunks.append(current) + current = piece + else: + current = candidate + if current: + chunks.append(current) + return chunks + + +def prepare_segments(text: str) -> list[tuple[str, str]]: + """Prepare short, deterministic German/English XTTS requests.""" + prepared: list[tuple[str, str]] = [] + for language, segment in segment_languages(clean_for_speech(text)): + if not re.search(r"\w", segment, flags=re.UNICODE): + if prepared: + previous_language, previous_text = prepared[-1] + prepared[-1] = (previous_language, previous_text + segment.strip()) + continue + spoken = segment + if language == "en": + spoken = ENGLISH_PRONUNCIATIONS.get(segment.strip().lower(), segment) + for chunk in _split_chunk(spoken): + prepared.append((language, chunk)) + return prepared + + def _speaker_conditioning() -> dict: global SPEAKER_CONDITIONING with SPEAKER_LOCK: @@ -180,6 +294,70 @@ def _xtts_pcm(text: str, language: str, conditioning: dict) -> bytes: return audio[44:] +def _trim_pcm(pcm: bytes) -> bytes: + """Trim generated edge silence while retaining a small safety padding.""" + samples = array("h") + samples.frombytes(pcm) + if not samples: + return pcm + first = next((i for i, value in enumerate(samples) + if abs(value) >= TRIM_THRESHOLD), 0) + last = next((i for i in range(len(samples) - 1, -1, -1) + if abs(samples[i]) >= TRIM_THRESHOLD), len(samples) - 1) + padding = int(24000 * max(0, TRIM_PADDING_MS) / 1000) + first = max(0, first - padding) + last = min(len(samples) - 1, last + padding) + return samples[first:last + 1].tobytes() + + +def _fade_edge(pcm: bytes, *, fade_in: bool = False, + fade_out: bool = False) -> bytes: + samples = array("h") + samples.frombytes(pcm) + count = min(len(samples), int(24000 * max(0, CROSSFADE_MS) / 1000)) + if count <= 1: + return pcm + if fade_in: + for index in range(count): + samples[index] = int(samples[index] * index / (count - 1)) + if fade_out: + start = len(samples) - count + for index in range(count): + samples[start + index] = int( + samples[start + index] * (count - 1 - index) / (count - 1)) + return samples.tobytes() + + +def _join_pcm(parts: list[tuple[str, bytes]]) -> bytes: + """Join clips without clicks; pause only at real sentence boundaries.""" + if not parts: + return b"" + output = bytearray() + sentence_silence = b"\x00\x00" * int( + 24000 * max(0, SENTENCE_PAUSE_MS) / 1000) + for index, (text, pcm) in enumerate(parts): + pcm = _trim_pcm(pcm) + previous_ends_sentence = index > 0 and bool( + re.search(r"[.!?;:]\s*$", parts[index - 1][0])) + if index == 0: + output.extend(_fade_edge(pcm, fade_in=True)) + elif previous_ends_sentence: + if output: + faded = _fade_edge(bytes(output), fade_out=True) + output[:] = faded + output.extend(sentence_silence) + output.extend(_fade_edge(pcm, fade_in=True)) + else: + # Language switches inside a sentence get no artificial pause. + # Small fades remove the discontinuity that otherwise sounds like + # a high click or beep between independently generated clips. + if output: + faded = _fade_edge(bytes(output), fade_out=True) + output[:] = faded + output.extend(_fade_edge(pcm, fade_in=True)) + return bytes(output) + + def _wav(pcm: bytes) -> bytes: output = io.BytesIO() with wave.open(output, "wb") as wav_file: @@ -211,19 +389,14 @@ def _convert(wav_bytes: bytes, output_format: str, speed: float) -> tuple[bytes, def synthesize_xtts(text: str, output_format: str, speed: float) -> tuple[bytes, str]: conditioning = _speaker_conditioning() - pcm_parts: list[bytes] = [] - silence = b"\x00\x00" * int(24000 * max(0, SILENCE_MS) / 1000) - for language, segment in segment_languages(text): + pcm_parts: list[tuple[str, bytes]] = [] + for language, segment in prepare_segments(text): if not segment.strip(): continue - pcm_parts.append(_xtts_pcm(segment, language, conditioning)) - if silence: - pcm_parts.append(silence) - if pcm_parts and silence: - pcm_parts.pop() + pcm_parts.append((segment, _xtts_pcm(segment, language, conditioning))) if not pcm_parts: raise RuntimeError("no speech segments generated") - return _convert(_wav(b"".join(pcm_parts)), output_format, speed) + return _convert(_wav(_join_pcm(pcm_parts)), output_format, speed) def synthesize_piper(text: str, output_format: str,