michael/Athena-Deck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow readable profile names and configurable KV cache

Browse Source
This commit is contained in:
Mikei386
2026-09-30 09:52:36 +02:00
parent fac4a12d52
commit cbcd5a0131
7 changed files with 32 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inference.py
+1 -1
View File
@@ -177,7 +177,7 @@ class LlamaWorker:
env.update(CUDA_VISIBLE_DEVICES=','.join(g['uuid'] for g in selected),HF_HUB_OFFLINE='1',OMP_NUM_THREADS=str(params['threads']))
reserve_mode=params.get('gpu_reserve_mode','auto')
margins=[0 if reserve_mode=='none' else params.get('gpu_reserve_mib',{}).get(g['uuid'],512) if reserve_mode=='manual' else max(512,round(g['total_mib']*.025)) for g in selected]
common=['--model',str(model),'--ctx-size',str(params['context']),'--parallel',str(params['slots']),'--batch-size',str(params['batch']),'--ubatch-size',str(params['ubatch']),'--cache-type-k','q4_0','--cache-type-v','q4_0','--flash-attn','on','--split-mode',params['split_mode'],'--fit-target',','.join(map(str,margins))]
common=['--model',str(model),'--ctx-size',str(params['context']),'--parallel',str(params['slots']),'--batch-size',str(params['batch']),'--ubatch-size',str(params['ubatch']),'--cache-type-k',params.get('cache_type_k','q4_0'),'--cache-type-v',params.get('cache_type_v','q4_0'),'--flash-attn','on','--split-mode',params['split_mode'],'--fit-target',','.join(map(str,margins))]
if params['tensor_split']:common+=['--tensor-split',','.join(map(str,params['tensor_split']))]
tool=str(directory/'build/bin/llama-fit-params')
mtp_fit=['--deck-mtp'] if mtp else []