michael/Athena-Deck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add internal chat testing with memory diagnostics and cancellation

Browse Source
This commit is contained in:
Mikei386
2026-09-28 20:43:40 +02:00
parent ff5d36252a
commit 7ada7f6f20
15 changed files with 265 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inference.py
+35 -15
View File
@@ -72,7 +72,7 @@ class Scheduler:
class LlamaWorker:
def __init__(self,root,catalog,runtime):
self.root=Path(root);self.catalog=catalog;self.runtime=runtime;self.lock=threading.RLock()
self.process=None;self.fit_process=None;self.generation=0;self.port=None;self.profile=None;self.state='stopped';self.error=None;self.key=None;self.devices=[]
self.process=None;self.fit_process=None;self.generation=0;self.port=None;self.profile=None;self.state='stopped';self.error=None;self.key=None;self.devices=[];self.memory_plan=None;self.phase='Gestoppt';self.oom_before=0
def build(self):
state=self.runtime.status()
build=next((b for b in state['builds'] if b['id']==state['active']),None)
@@ -88,10 +88,17 @@ class LlamaWorker:
if not p.get('model') or not p['model']['file'].endswith('.gguf'):raise InferenceError('Ein lokales GGUF-Modell wird benötigt.')
return []
except ValueError as exc:return [str(exc)]
@staticmethod
def oom_count():
try:return int(dict(line.split() for line in Path('/sys/fs/cgroup/memory.events').read_text().splitlines()).get('oom_kill',0))
except (OSError,ValueError):return 0
def crash_message(self):
if self.oom_count()>self.oom_before:return 'OOM-Kill im Deck-RAM-Bereich erkannt. Das RAM-Limit wurde überschritten; Kontext oder Modellgröße reduzieren.'
return 'Modellprozess beendet. GPU-OOM oder Modell-/Buildfehler möglich; Ursache nicht eindeutig bestätigt. Kontext oder Microbatch reduzieren und erneut testen.'
def status(self):
with self.lock:
if self.process and self.process.poll() is not None and self.state=='ready':self.state='failed';self.error='llama.cpp wurde unerwartet beendet.'
return dict(state=self.state,profile_id=self.profile['id'] if self.profile else None,profile_name=self.profile['name'] if self.profile else None,port=self.port,error=self.error,gpus=list(self.devices))
if self.process and self.process.poll() is not None and self.state=='ready':self.state='failed';self.error=self.crash_message()
return dict(state=self.state,profile_id=self.profile['id'] if self.profile else None,profile_name=self.profile['name'] if self.profile else None,port=self.port,error=self.error,gpus=list(self.devices),memory_plan=self.memory_plan,phase=self.phase)
def stop(self):
with self.lock:
self.generation+=1
@@ -106,18 +113,18 @@ class LlamaWorker:
try:p.wait(timeout=10)
except subprocess.TimeoutExpired:os.killpg(p.pid,signal.SIGKILL);p.wait(timeout=5)
except ProcessLookupError:pass
self.process=None;self.profile=None;self.port=None;self.state='stopped';self.devices=[];self.key=None
self.process=None;self.profile=None;self.port=None;self.state='stopped';self.phase='Gestoppt';self.devices=[];self.key=None
(self.root/'worker.key').unlink(missing_ok=True)
def ensure(self,profile):
def ensure(self,profile,cancel=lambda:False):
with self.lock:
if self.process and self.process.poll() is None and self.profile and (self.profile['id'],self.profile['revision'])==(profile['id'],profile['revision']):return
self.stop();self.state='loading';self.error=None;generation=self.generation
try:self._start(profile,generation)
self.stop();self.state='loading';self.error=None;self.memory_plan=None;self.phase='Speicherprüfung';self.oom_before=self.oom_count();generation=self.generation
try:self._start(profile,generation,cancel)
except Exception as exc:
self.stop()
with self.lock:self.state='failed';self.error=str(exc) if isinstance(exc,ValueError) else 'llama.cpp konnte nicht gestartet werden.'
with self.lock:self.state='failed';self.phase='Modellstart fehlgeschlagen';self.error=str(exc) if isinstance(exc,ValueError) else 'llama.cpp konnte nicht gestartet werden.'
raise InferenceError(self.error) from None
def _start(self,profile,generation):
def _start(self,profile,generation,cancel=lambda:False):
directory=self.build();params=profile['parameters'];entry=self.catalog.entry(profile['model_id'])
model=(self.catalog.root/entry['id']/('model'+Path(entry['file']).suffix)).resolve()
devices=probe();ids=params['gpu_devices']
@@ -141,7 +148,8 @@ class LlamaWorker:
tool=str(directory/'build/bin/llama-fit-params')
margins=[max(1024,round(g['total_mib']*.05)) for g in selected]
def estimate(layers,extra=()):
output=self._fit_command([tool]+common+['--gpu-layers',str(layers),'--fit-print','on']+list(extra),env,generation)
if cancel():raise InferenceError('Modellstart abgebrochen.')
output=self._fit_command([tool]+common+['--gpu-layers',str(layers),'--fit-print','on']+list(extra),env,generation,cancel)
rows={}
for line in output.splitlines():
parts=line.split()
@@ -150,6 +158,7 @@ class LlamaWorker:
gpu_fits=all(rows['CUDA'+str(i)]+margins[i]<=g['free_mib'] for i,g in enumerate(selected))
need=max(staging,rows['Host']*1024**2+4*GIB)
host_fits=mem.get('MemAvailable',0)>=need+4*GIB and (headroom is None or headroom>=need)
with self.lock:self.memory_plan=dict(profile_name=profile['name'],estimated=True,gpu_layers=layers,context=params['context'],slots=params['slots'],fits=gpu_fits and host_fits,host_required_mib=round(need/1024**2),host_available_mib=round(min(mem.get('MemAvailable',0)-4*GIB,headroom if headroom is not None else mem.get('MemAvailable',0))/1024**2),gpus=[dict(name=g.get('name',g['uuid']),uuid=g['uuid'],free_mib=g['free_mib'],required_mib=rows['CUDA'+str(i)],reserve_mib=margins[i]) for i,g in enumerate(selected)])
return gpu_fits and host_fits,rows
extra=[]
if params['tensor_split']:
@@ -165,8 +174,9 @@ class LlamaWorker:
else:high=middle-1
if best is None:raise InferenceError('Kontext und fester GPU-Split passen nicht sicher in GPU/RAM.')
layers,memory=best
estimate(layers)
else:
output=self._fit_command([tool]+common,env,generation)
output=self._fit_command([tool]+common,env,generation,cancel)
flags=shlex.split(output.strip());fit={}
if len(flags)%2:raise InferenceError('Fit-Werkzeug lieferte ungültige Parameter.')
for i in range(0,len(flags),2):
@@ -180,6 +190,8 @@ class LlamaWorker:
extra=['--tensor-split',fit['-ts']]
fits,memory=estimate(layers,extra)
if not fits:raise InferenceError('Speicherprognose überschreitet die GPU-/RAM-Reserve.')
if cancel():raise InferenceError('Modellstart abgebrochen.')
with self.lock:self.phase='Modell wird geladen'
launch=common+extra+['--gpu-layers',str(layers),'--fit','off','--kv-unified','--threads',str(params['threads']),'--load-mode','none','--host','127.0.0.1','--alias',profile['name'],'--no-webui','--log-disable']
self.root.mkdir(parents=True,exist_ok=True,mode=0o700)
with socket.socket() as sock:sock.bind(('127.0.0.1',0));port=sock.getsockname()[1]
@@ -193,27 +205,35 @@ class LlamaWorker:
self.profile=profile;self.port=port;self.key=key;self.devices=[g['uuid'] for g in selected]
deadline=time.monotonic()+300
while time.monotonic()<deadline:
if cancel():raise InferenceError('Modellstart abgebrochen.')
with self.lock:
if not self.process or self.process.poll() is not None:raise InferenceError('llama.cpp-Start fehlgeschlagen; Modell/Build oder Speicher passt nicht.')
if not self.process or self.process.poll() is not None:raise InferenceError(self.crash_message())
current={g['uuid']:g for g in probe()}
if any(current.get(g['uuid'],{}).get('processes',2)>1 for g in selected):raise InferenceError('Ein weiterer Prozess verwendet die reservierte GPU. Deck bricht seinen Start ab.')
conn=http.client.HTTPConnection('127.0.0.1',port,timeout=2)
try:
conn.request('GET','/health',headers={'Authorization':'Bearer '+key});r=conn.getresponse()
if r.status==200:
with self.lock:self.state='ready'
with self.lock:self.state='ready';self.phase='Modell bereit'
threading.Thread(target=self._monitor,args=(generation,),daemon=True).start()
return
except OSError:pass
finally:conn.close()
time.sleep(1)
raise InferenceError('llama.cpp wurde nicht rechtzeitig bereit.')
def _fit_command(self,args,env,generation):
def _fit_command(self,args,env,generation,cancel=lambda:False):
with self.lock:
if self.generation!=generation:raise InferenceError('Modellstart abgebrochen.')
process=subprocess.Popen(args,env=env,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True)
self.fit_process=process
try:output,_=process.communicate(timeout=180)
try:
deadline=time.monotonic()+180
while True:
if cancel():
process.terminate();process.communicate(timeout=3);raise InferenceError('Modellstart abgebrochen.')
try:output,_=process.communicate(timeout=1);break
except subprocess.TimeoutExpired:
if time.monotonic()>=deadline:raise
except subprocess.TimeoutExpired:
process.kill();process.communicate();raise InferenceError('Zeitlimit der Speicher-Einpassung überschritten.') from None
finally: