michael/Athena-Deck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add configurable MTP and shared-weight draft memory fitting

Browse Source
This commit is contained in:
Mikei386
2026-09-28 20:59:43 +02:00
parent ec62c859f7
commit 13ed059aaf
7 changed files with 87 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inference.py
+6 -2
View File
@@ -127,6 +127,8 @@ class LlamaWorker:
def _start(self,profile,generation,cancel=lambda:False):
directory=self.build();params=profile['parameters'];entry=self.catalog.entry(profile['model_id'])
model=(self.catalog.root/entry['id']/('model'+Path(entry['file']).suffix)).resolve()
mtp=params.get('mtp',False)
if mtp and not (directory/'build/bin/deck-mtp-fit-v1').is_file():raise InferenceError('MTP benötigt einen neu gebauten llama.cpp-Build mit Deck-MTP-Speicherprüfung. Unter Laufzeiten erneut bauen.')
devices=probe();ids=params['gpu_devices']
if ids:
selected=[next((g for g in devices if g['uuid']==ident),None) for ident in ids]
@@ -146,10 +148,11 @@ class LlamaWorker:
common=['--model',str(model),'--ctx-size',str(params['context']),'--parallel',str(params['slots']),'--batch-size',str(params['batch']),'--ubatch-size',str(params['ubatch']),'--cache-type-k','q4_0','--cache-type-v','q4_0','--flash-attn','on','--split-mode',params['split_mode'],'--fit-target',','.join(str(max(1024,round(g['total_mib']*.05))) for g in selected)]
if params['tensor_split']:common+=['--tensor-split',','.join(map(str,params['tensor_split']))]
tool=str(directory/'build/bin/llama-fit-params')
mtp_fit=['--deck-mtp'] if mtp else []
margins=[max(1024,round(g['total_mib']*.05)) for g in selected]
def estimate(layers,extra=()):
if cancel():raise InferenceError('Modellstart abgebrochen.')
output=self._fit_command([tool]+common+['--gpu-layers',str(layers),'--fit-print','on']+list(extra),env,generation,cancel)
output=self._fit_command([tool]+mtp_fit+common+['--gpu-layers',str(layers),'--fit-print','on']+list(extra),env,generation,cancel)
rows={}
for line in output.splitlines():
parts=line.split()
@@ -176,7 +179,7 @@ class LlamaWorker:
layers,memory=best
estimate(layers)
else:
output=self._fit_command([tool]+common,env,generation,cancel)
output=self._fit_command([tool]+mtp_fit+common,env,generation,cancel)
flags=shlex.split(output.strip());fit={}
if len(flags)%2:raise InferenceError('Fit-Werkzeug lieferte ungültige Parameter.')
for i in range(0,len(flags),2):
@@ -193,6 +196,7 @@ class LlamaWorker:
if cancel():raise InferenceError('Modellstart abgebrochen.')
with self.lock:self.phase='Modell wird geladen'
launch=common+extra+['--gpu-layers',str(layers),'--fit','off','--kv-unified','--threads',str(params['threads']),'--load-mode','none','--host','127.0.0.1','--alias',profile['name'],'--no-webui','--log-disable']
if mtp:launch+=['--spec-type','draft-mtp','--spec-draft-n-max',str(params.get('mtp_tokens',2)),'--spec-draft-p-min',str(params.get('mtp_min_p',.05)),'--spec-draft-type-k','f16','--spec-draft-type-v','f16']
self.root.mkdir(parents=True,exist_ok=True,mode=0o700)
with socket.socket() as sock:sock.bind(('127.0.0.1',0));port=sock.getsockname()[1]
key=secrets.token_urlsafe(32);keypath=self.root/'worker.key'