Keep TTS and STT workers resident between requests
This commit is contained in:
Mikei386 committed
1 parent
3dcf2d3f76
commit
90a97567f3
11 files changed
+197
-80
No files matched your search
@@ -50,7 +50,7 @@ def read_upload(handler):
|
||||
|
||||
class STT:
|
||||
def __init__(self,profiles,catalog,runtime):
|
||||
self.profiles=profiles;self.catalog=catalog;self.runtime=runtime;self.lock=threading.RLock();self.job=None;self.process=None;self.cancel=threading.Event()
|
||||
self.profiles=profiles;self.catalog=catalog;self.runtime=runtime;self.lock=threading.RLock();self.job=None;self.process=None;self.port=None;self.key=None;self.binary_path=None;self.cancel=threading.Event()
|
||||
def build(self):
|
||||
state=self.runtime.status();ident=state.get('active')
|
||||
if not isinstance(ident,str) or len(ident)!=32 or any(c not in '0123456789abcdef' for c in ident):raise ValueError('Zuerst unter Laufzeiten → llama.cpp einen Build erstellen und aktivieren.')
|
||||
@@ -75,7 +75,7 @@ class STT:
|
||||
except ValueError:projector=False
|
||||
try:self.model_entry();model=True
|
||||
except ValueError:model=False
|
||||
with self.lock:return dict(model=model,job=dict(self.job) if self.job else None,installed=installed,projector=projector,repo=REPO,revision=REVISION)
|
||||
with self.lock:return dict(model=model,job=dict(self.job) if self.job else None,installed=installed,projector=projector,repo=REPO,revision=REVISION,loaded=bool(self.process and self.process.poll() is None))
|
||||
def model_entry(self):
|
||||
for x in self.catalog.status()['entries']:
|
||||
if supported(x):return self.catalog.entry(x['id'])
|
||||
@@ -102,30 +102,39 @@ class STT:
|
||||
if self.job and self.job['state']=='running':raise ValueError('Ein STT-Auftrag läuft bereits.')
|
||||
p=next((p for p in self.profiles.status()['profiles'] if p['id']==profile_id and p['kind']=='stt'),None)
|
||||
if not p or not p['runnable']:raise ValueError('STT-Profil nicht ausführbar. Zuerst Einrichten öffnen.')
|
||||
headroom=cgroup_headroom()
|
||||
if headroom is not None and headroom<4*1024**3:raise ValueError('Mindestens 4 GiB freier Deck-RAM werden benötigt.')
|
||||
if not self.process or self.process.poll() is not None:
|
||||
headroom=cgroup_headroom()
|
||||
if headroom is not None and headroom<4*1024**3:raise ValueError('Mindestens 4 GiB freier Deck-RAM werden benötigt.')
|
||||
binary=self.build();model=self.catalog.root/p['model_id']/'model.gguf';projector=self.catalog.root/self.projector()['id']/'model.gguf'
|
||||
self.cancel.clear();ident=uuid.uuid4().hex;self.job=dict(id=ident,state='running',phase='Spracherkennung lädt auf der CPU',profile_id=profile_id)
|
||||
self.cancel.clear();ident=uuid.uuid4().hex;self.job=dict(id=ident,state='running',phase='Vorhandene Spracherkennung wird verwendet' if self.process and self.process.poll() is None else 'Spracherkennung lädt auf der CPU',profile_id=profile_id)
|
||||
threading.Thread(target=self._run,args=(binary,model,projector,audio,language),daemon=True).start();return dict(self.job)
|
||||
def _run(self,binary,model,projector,audio,language):
|
||||
process=None;result=dict(state='failed',phase='Spracherkennung fehlgeschlagen. Laufzeit und Speicher prüfen.')
|
||||
try:
|
||||
key=secrets.token_hex(24)
|
||||
with socket.socket() as s:s.bind(('127.0.0.1',0));port=s.getsockname()[1]
|
||||
env=dict(os.environ,CUDA_VISIBLE_DEVICES='',OMP_NUM_THREADS='2')
|
||||
args=[str(binary),'--model',str(model),'--mmproj',str(projector),'--no-mmproj-offload','--n-gpu-layers','0','--ctx-size','4096','--threads','2','--parallel','1','--host','127.0.0.1','--port',str(port),'--no-ui','--fit','off','--alias','deck-stt','--api-key',key]
|
||||
with self.lock:
|
||||
if self.cancel.is_set():raise InterruptedError()
|
||||
process=subprocess.Popen(args,env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True);self.process=process
|
||||
process=self.process if self.process and self.process.poll() is None and self.binary_path==str(binary) else None
|
||||
if process is None:
|
||||
self.unload_idle()
|
||||
key=secrets.token_hex(24)
|
||||
with socket.socket() as s:s.bind(('127.0.0.1',0));port=s.getsockname()[1]
|
||||
env=dict(os.environ,CUDA_VISIBLE_DEVICES='',OMP_NUM_THREADS='2')
|
||||
args=[str(binary),'--model',str(model),'--mmproj',str(projector),'--no-mmproj-offload','--n-gpu-layers','0','--ctx-size','4096','--threads','2','--parallel','1','--host','127.0.0.1','--port',str(port),'--no-ui','--fit','off','--alias','deck-stt','--api-key',key]
|
||||
with self.lock:
|
||||
if self.cancel.is_set():raise InterruptedError()
|
||||
process=subprocess.Popen(args,env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True);self.process=process;self.port=port;self.key=key;self.binary_path=str(binary)
|
||||
deadline=time.monotonic()+90
|
||||
while True:
|
||||
if self.cancel.wait(.3):raise InterruptedError()
|
||||
if process.poll() is not None:raise ValueError('llama.cpp konnte das ASR-Modell nicht laden. Build-Unterstützung und freien RAM prüfen.')
|
||||
if time.monotonic()>deadline:raise ValueError('Zeitlimit beim Laden des ASR-Modells.')
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(f'http://127.0.0.1:{port}/health',headers={'Authorization':'Bearer '+key}),timeout=1) as response:
|
||||
if response.status==200:break
|
||||
except OSError:continue
|
||||
else:
|
||||
with self.lock:port=self.port;key=self.key
|
||||
base=f'http://127.0.0.1:{port}';headers={'Authorization':'Bearer '+key};deadline=time.monotonic()+90
|
||||
while True:
|
||||
if self.cancel.wait(.3):raise InterruptedError()
|
||||
if process.poll() is not None:raise ValueError('llama.cpp konnte das ASR-Modell nicht laden. Build-Unterstützung und freien RAM prüfen.')
|
||||
if time.monotonic()>deadline:raise ValueError('Zeitlimit beim Laden des ASR-Modells.')
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(base+'/health',headers=headers),timeout=1) as response:
|
||||
if response.status==200:break
|
||||
except OSError:continue
|
||||
with self.lock:self.job['phase']='Audio wird auf der CPU transkribiert'
|
||||
boundary='deck-'+uuid.uuid4().hex
|
||||
body=f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.wav"\r\nContent-Type: audio/wav\r\n\r\n'.encode()+audio+b'\r\n'
|
||||
@@ -135,24 +144,26 @@ class STT:
|
||||
payload=json.loads(response.read(1024*1024))
|
||||
if not isinstance(payload.get('text'),str):raise ValueError('Die Laufzeit hat kein Transkript geliefert.')
|
||||
text=payload['text'].split('<asr_text>')[-1].replace('<|endoftext|>','').strip()
|
||||
result=dict(state='complete',phase='Transkription fertig · Modell entladen',text=text)
|
||||
result=dict(state='complete',phase='Transkription fertig · Modell bleibt geladen',text=text)
|
||||
except InterruptedError:result=dict(state='cancelled',phase='Transkription abgebrochen')
|
||||
except ValueError as exc:result=dict(state='failed',phase=str(exc))
|
||||
except Exception:pass
|
||||
finally:
|
||||
if process and process.poll() is None:
|
||||
try:
|
||||
os.killpg(process.pid,signal.SIGTERM)
|
||||
try:process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:os.killpg(process.pid,signal.SIGKILL);process.wait()
|
||||
except ProcessLookupError:pass
|
||||
if result['state']!='complete':self.unload_idle()
|
||||
with self.lock:
|
||||
if self.cancel.is_set():result=dict(state='cancelled',phase='Transkription abgebrochen')
|
||||
self.process=None;self.job.update(result)
|
||||
self.job.update(result)
|
||||
def unload_idle(self):
|
||||
with self.lock:
|
||||
process=self.process;self.process=None;self.port=None;self.key=None;self.binary_path=None
|
||||
if process and process.poll() is None:
|
||||
try:
|
||||
os.killpg(process.pid,signal.SIGTERM)
|
||||
try:process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:os.killpg(process.pid,signal.SIGKILL);process.wait()
|
||||
except ProcessLookupError:pass
|
||||
def stop(self):
|
||||
with self.lock:
|
||||
self.cancel.set()
|
||||
if self.process and self.process.poll() is None:
|
||||
try:os.killpg(self.process.pid,signal.SIGTERM)
|
||||
except ProcessLookupError:pass
|
||||
self.unload_idle()
|
||||
return dict(cancellation_requested=True)
|
||||
Reference in new issue
Block a user