michael/Athena-Deck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expose model capability hints and configurable vision projector placement

Browse Source
This commit is contained in:
Mikei386
2026-09-28 22:03:39 +02:00
parent a36a151c94
commit 06c6292cf3
10 changed files with 112 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inference.py
+18 -2
View File
@@ -85,6 +85,9 @@ class LlamaWorker:
def blockers(self,p):
try:
self.build()
if p['parameters'].get('vision_projector'):
projector=self.catalog.entry(p['parameters']['vision_projector'])
if projector.get('role')!='vision_projector':raise InferenceError('Vision-Projektor fehlt oder ist ungültig.')
if not p.get('model') or not p['model']['file'].endswith('.gguf'):raise InferenceError('Ein lokales GGUF-Modell wird benötigt.')
return []
except ValueError as exc:return [str(exc)]
@@ -141,7 +144,17 @@ class LlamaWorker:
if params['split_mode']=='none':selected=selected[:1]
if not selected or any(g['processes'] or g['free_mib']<2048 for g in selected):raise InferenceError('Benötigte GPU ist durch einen anderen Dienst belegt. Deck stoppt keine fremden Prozesse.')
mem={line.split(':')[0]:int(line.split()[1])*1024 for line in Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemAvailable:')}
staging=entry['size']+4*GIB
projector=None;vision_args=[];vision_bytes=0;vision_device=params.get('vision_device','cpu')
if params.get('vision_projector'):
projector=self.catalog.entry(params['vision_projector'])
if projector.get('role')!='vision_projector':raise InferenceError('Kein gültiger Vision-Projektor.')
vision_bytes=projector['size']+2*GIB
vision_args=['--mmproj',str((self.catalog.root/projector['id']/'model.gguf').resolve())]
if vision_device=='cpu':vision_args+=['--no-mmproj-offload']
else:
if vision_device not in [g['uuid'] for g in selected]:raise InferenceError('Projektor-GPU ist nicht ausgewählt.')
vision_args+=['--mmproj-device','CUDA'+str([g['uuid'] for g in selected].index(vision_device)),'--mmproj-offload']
staging=entry['size']+4*GIB+vision_bytes
if mem.get('MemAvailable',0)<staging+4*GIB:raise InferenceError('Zu wenig freier RAM für Modellladen und Host-Reserve.')
headroom=cgroup_headroom()
if headroom is not None and headroom<staging:raise InferenceError('Deck-RAM-Limit reicht für das Laden dieses Modells nicht aus.')
@@ -161,6 +174,9 @@ class LlamaWorker:
parts=line.split()
if len(parts)==4 and all(x.isdigit() for x in parts[1:]):rows[parts[0]]=sum(map(int,parts[1:]))
if 'Host' not in rows or any('CUDA'+str(i) not in rows for i in range(len(selected))):raise InferenceError('Speicherprognose unvollständig; Start abgebrochen.')
if projector:
target='Host' if vision_device=='cpu' else 'CUDA'+str([g['uuid'] for g in selected].index(vision_device))
rows[target]+=round(vision_bytes/1024**2)
gpu_fits=all(rows['CUDA'+str(i)]+margins[i]<=g['free_mib'] for i,g in enumerate(selected))
need=max(staging,rows['Host']*1024**2+4*GIB)
host_fits=mem.get('MemAvailable',0)>=need+4*GIB and (headroom is None or headroom>=need)
@@ -202,7 +218,7 @@ class LlamaWorker:
if cancel():raise InferenceError('Modellstart abgebrochen.')
if plan_only:return
with self.lock:self.phase='Modell wird geladen'
launch=common+extra+['--gpu-layers',str(layers),'--fit','off','--kv-unified','--threads',str(params['threads']),'--load-mode','none','--host','127.0.0.1','--alias',profile['name'],'--no-webui','--log-disable']
launch=common+extra+vision_args+['--gpu-layers',str(layers),'--fit','off','--kv-unified','--threads',str(params['threads']),'--load-mode','none','--host','127.0.0.1','--alias',profile['name'],'--no-webui','--log-disable']
if mtp:launch+=['--spec-type','draft-mtp','--spec-draft-n-max',str(params.get('mtp_tokens',2)),'--spec-draft-p-min',str(params.get('mtp_min_p',.05)),'--spec-draft-type-k','f16','--spec-draft-type-v','f16']
self.root.mkdir(parents=True,exist_ok=True,mode=0o700)
with socket.socket() as sock:sock.bind(('127.0.0.1',0));port=sock.getsockname()[1]