Keep TTS and STT workers resident between requests

tts_worker.py

@@ -1,21 +1,30 @@
-"""One isolated offline Qwen-TTS request; stdin is never persisted."""
+"""Persistent offline Qwen-TTS worker; line-delimited stdin, no prompt logging."""
 import json,sys
 from pathlib import Path
 
 def main():
- request=json.load(sys.stdin);root=Path(request['output'])
+ model_path=sys.argv[1]
  try:
   import torch,soundfile as sf
   from qwen_tts import Qwen3TTSModel
-  model=Qwen3TTSModel.from_pretrained(request['model'],device_map='cuda:0',dtype=torch.bfloat16,attn_implementation='sdpa',local_files_only=True)
-  waves,rate=model.generate_custom_voice(text=request['text'],language=request['language'],speaker=request['speaker'],max_new_tokens=1024)
-  wave=waves[0]
-  if request['speed']!=1:
-   import librosa
-   wave=librosa.effects.time_stretch(wave,rate=request['speed'])
-  sf.write(root/'result.wav',wave,rate,subtype='PCM_16')
-  (root/'result.json').write_text(json.dumps(dict(ok=True,sample_rate=rate,duration=len(wave)/rate)))
+  model=Qwen3TTSModel.from_pretrained(model_path,device_map='cuda:0',dtype=torch.bfloat16,attn_implementation='sdpa',local_files_only=True)
  except Exception as exc:
-  (root/'result.json').write_text(json.dumps(dict(ok=False,error='CUDA-Speicher reicht nicht aus.' if 'outofmemory' in type(exc).__name__.lower() or 'out of memory' in str(exc).lower() else 'TTS-Ausführung fehlgeschlagen: '+type(exc).__name__)))
-  sys.exit(1)
-if __name__=='__main__':main()
+  print(json.dumps(dict(ready=False,error='CUDA-Speicher reicht nicht aus.' if 'outofmemory' in type(exc).__name__.lower() or 'out of memory' in str(exc).lower() else 'TTS-Modell konnte nicht geladen werden.')),flush=True)
+  return 1
+ print(json.dumps(dict(ready=True)),flush=True)
+ for line in sys.stdin:
+  root=None
+  try:
+   request=json.loads(line);root=Path(request['output'])
+   waves,rate=model.generate_custom_voice(text=request['text'],language=request['language'],speaker=request['speaker'],max_new_tokens=1024)
+   wave=waves[0]
+   if request['speed']!=1:
+    import librosa
+    wave=librosa.effects.time_stretch(wave,rate=request['speed'])
+   sf.write(root/'result.wav',wave,rate,subtype='PCM_16')
+   (root/'result.json').write_text(json.dumps(dict(ok=True,sample_rate=rate,duration=len(wave)/rate)))
+  except Exception as exc:
+   if root is not None:
+    (root/'result.json').write_text(json.dumps(dict(ok=False,error='CUDA-Speicher reicht nicht aus.' if 'outofmemory' in type(exc).__name__.lower() or 'out of memory' in str(exc).lower() else 'TTS-Ausführung fehlgeschlagen: '+type(exc).__name__)))
+ return 0
+if __name__=='__main__':sys.exit(main())