Add bounded context ranges and complete GPU split coverage to auto test
This commit is contained in:
Mikei386
+44
-27
@@ -5,6 +5,11 @@ from profiles import SCHEMAS,CHAT_GPU_DEFAULTS
|
||||
from image_test import probe
|
||||
from inference import InferenceError
|
||||
|
||||
CONTEXT_STEPS=(2048,4096,8192,16384,32768,65536,131072,160000,192000,262144)
|
||||
MAX_CANDIDATES=300
|
||||
MAX_SECONDS=7200
|
||||
GPU_SPLITS=([98,2],[95,5],[90,10],[85,15],[75,25],[50,50])
|
||||
|
||||
def model_layers(path,mtp):
|
||||
"""Read bounded GGUF metadata only; never load tensors or tokenizer strings."""
|
||||
import struct
|
||||
@@ -68,14 +73,16 @@ class AutoTests:
|
||||
def update(self,**kw):
|
||||
with self.lock:self.job.update(kw);self.persist()
|
||||
def start(self,data):
|
||||
if set(data)!={'model_id','max_context','mtp'} or type(data['max_context'])!=int or data['max_context'] not in (4096,8192,16384,32768,65536,131072,160000,192000,262144) or type(data['mtp'])!=bool:raise ValueError('Modell, Kontextgrenze und MTP auswählen.')
|
||||
if set(data) not in ({'model_id','max_context','mtp'},{'model_id','start_context','max_context','mtp'}):raise ValueError('Modell, Start- und Endkontext sowie MTP auswählen.')
|
||||
start_context=data.get('start_context',2048)
|
||||
if type(start_context)!=int or start_context not in CONTEXT_STEPS or type(data['max_context'])!=int or data['max_context'] not in CONTEXT_STEPS or start_context>data['max_context'] or type(data['mtp'])!=bool:raise ValueError('Start- und Endkontext müssen gültige Stufen in aufsteigender Reihenfolge sein.')
|
||||
model=self.worker.catalog.entry(data['model_id'])
|
||||
if model['kind']!='chat' or not model['file'].endswith('.gguf') or not model['profile_eligible']:raise ValueError('Ein heruntergeladenes Chat-GGUF auswählen.')
|
||||
devices=probe();primary=next((g for g in devices if '5080' in g['name']),None);secondary=next((g for g in devices if '3060' in g['name']),None)
|
||||
if not primary or not secondary:raise ValueError('Dieser Auto-Test benötigt RTX 5080 und RTX 3060. Keine stillschweigende andere GPU-Zuordnung.')
|
||||
with self.lock:
|
||||
if self.thread and self.thread.is_alive():raise ValueError('Ein Auto-Test läuft bereits.')
|
||||
self.cancel.clear();self.job=dict(id=uuid.uuid4().hex,state='running',phase='Wartet auf exklusive Modellreservierung',model_id=model['id'],model_file=model['file'],max_context=data['max_context'],mtp=data['mtp'],results=[],attempts=0,started_at=time.time(),error=None);self.persist()
|
||||
self.cancel.clear();self.job=dict(id=uuid.uuid4().hex,state='running',phase='Wartet auf exklusive Modellreservierung',model_id=model['id'],model_file=model['file'],start_context=start_context,max_context=data['max_context'],mtp=data['mtp'],results=[],attempts=0,max_candidates=MAX_CANDIDATES,completed_contexts=[],started_at=time.time(),error=None);self.persist()
|
||||
self.thread=threading.Thread(target=self.run,args=(primary['uuid'],secondary['uuid']),daemon=True);self.thread.start()
|
||||
return self.status()
|
||||
def stop(self):
|
||||
@@ -99,33 +106,39 @@ class AutoTests:
|
||||
with self.lock:self.conn=None
|
||||
def benchmark(self,context):
|
||||
# Tokenize synthetic text with this model; do not estimate token count from characters.
|
||||
target=int(context*.75);unit='alpha beta gamma delta epsilon zeta eta theta 0123456789. '
|
||||
# Keep room for 64 output tokens and template overhead, but verify nearly the full context.
|
||||
target=context-max(128,context//100);unit='alpha beta gamma delta epsilon zeta eta theta 0123456789. '
|
||||
text=unit*(target//4+1);tokens=self.request('/tokenize',{'content':text,'add_special':True})['tokens']
|
||||
if len(tokens)<target:raise InferenceError('Synthetischer Kontext konnte nicht gefüllt werden.')
|
||||
result=self.request('/completion',dict(prompt=tokens[:target],n_predict=64,temperature=0,seed=1234,cache_prompt=False,stream=False))
|
||||
timing=result.get('timings',{})
|
||||
if not timing.get('predicted_n') or timing.get('prompt_n',0)<target*.95:raise InferenceError('Keine vollständige Kontext-/Ausgabemessung zurückgegeben.')
|
||||
if not timing.get('predicted_n') or timing.get('prompt_n',0)<target*.99:raise InferenceError('Keine vollständige Kontext-/Ausgabemessung zurückgegeben.')
|
||||
return {k:timing[k] for k in ('prompt_n','prompt_ms','prompt_per_second','predicted_n','predicted_ms','predicted_per_second') if k in timing}
|
||||
def test_candidate(self,context,tier,devices,split,ratio,offload,layer_count=None):
|
||||
successes=[]
|
||||
for micro in (128,64,256):
|
||||
if self.cancel.is_set():raise InterruptedError()
|
||||
if self.job['attempts']>=96 or time.time()-self.job['started_at']>7200:
|
||||
raise TestBudget()
|
||||
if self.job['attempts']>=self.job.get('max_candidates',MAX_CANDIDATES):raise TestBudget('Kandidatenlimit erreicht')
|
||||
if time.time()-self.job['started_at']>MAX_SECONDS:raise TestBudget('Zeitlimit erreicht')
|
||||
params={**{k:v[2] for k,v in SCHEMAS['chat'].items()},**CHAT_GPU_DEFAULTS,'context':context,'slots':1,'batch':2048,'ubatch':micro,'gpu_devices':devices,'split_mode':split,'tensor_split':ratio,'gpu_offload':offload,'mtp':self.job['mtp']}
|
||||
p=dict(id='auto-'+self.job['id'],revision=self.job['attempts']+1,name='deck-auto-test',kind='chat',model_id=self.job['model_id'],parameters=params)
|
||||
self.update(attempts=p['revision'],phase=f'{context} Kontext · {tier} · Microbatch {micro}')
|
||||
row=dict(context=context,tier=tier,parameters=params,success=False,primary_gpu_layers=layer_count)
|
||||
self.worker.stop()
|
||||
stage='prediction'
|
||||
try:
|
||||
self.worker.plan(p,cancel=self.cancel.is_set)
|
||||
stage='model_start'
|
||||
self.worker.ensure(p,cancel=self.cancel.is_set)
|
||||
# Small warm-up, then populated-context measurement, bounded output.
|
||||
stage='measurement'
|
||||
self.request('/completion',dict(prompt='Synthetic warmup.',n_predict=16,temperature=0,cache_prompt=False))
|
||||
row.update(success=True,timings=self.benchmark(context),memory_plan=copy.deepcopy(self.worker.memory_plan),tested_context_fraction=.75)
|
||||
timings=self.benchmark(context)
|
||||
row.update(success=True,timings=timings,memory_plan=copy.deepcopy(self.worker.memory_plan),tested_context_fraction=timings.get('prompt_n',0)/context)
|
||||
successes.append(row)
|
||||
except Exception as exc:
|
||||
if self.cancel.is_set():raise InterruptedError()
|
||||
row['failure_stage']=stage
|
||||
row['error']=str(exc) if isinstance(exc,ValueError) else 'Testprozess oder Verbindung fehlgeschlagen; kein eindeutiger OOM-Nachweis.'
|
||||
finally:self.worker.stop()
|
||||
with self.lock:self.job['results'].append(row);self.persist()
|
||||
@@ -135,29 +148,33 @@ class AutoTests:
|
||||
try:
|
||||
with self.scheduler.lease(('auto-test',self.job['id']),timeout=0,allowed=lambda:not self.cancel.is_set()):
|
||||
try:
|
||||
contexts=[x for x in (2048,4096,8192,16384,32768,65536,131072,160000,192000,262144) if x<=self.job['max_context']]
|
||||
contexts=[x for x in CONTEXT_STEPS if self.job.get('start_context',2048)<=x<=self.job['max_context']]
|
||||
fine_search=[]
|
||||
for context in contexts:
|
||||
candidates=[('5080',[primary],'none',[],'full')]+[('5080 + 3060',[primary,secondary],'layer',s,'full') for s in ([95,5],[90,10],[85,15],[75,25],[50,50])]+[('5080 + 3060 + CPU',[primary,secondary],'layer',[85,15],'auto')]
|
||||
found=False;upper=100
|
||||
for tier,devices,split,ratio,offload in candidates:
|
||||
successes=self.test_candidate(context,tier,devices,split,ratio,offload)
|
||||
if successes:
|
||||
found=True
|
||||
if tier=='5080 + 3060':
|
||||
try:
|
||||
entry=self.worker.catalog.entry(self.job['model_id'])
|
||||
total=model_layers(self.worker.catalog.root/entry['id']/('model'+Path(entry['file']).suffix),self.job['mtp'])
|
||||
except (OSError,ValueError,TypeError) as exc:
|
||||
self.update(fine_search_note='Layer-Feinsuche nicht verfügbar: '+str(exc));break
|
||||
for layers,finer in fine_splits(total,ratio,upper):
|
||||
result=self.test_candidate(context,'5080 + 3060 · Layer-Feinsuche',devices,split,finer,offload,layers)
|
||||
if not result:break
|
||||
break
|
||||
if tier=='5080 + 3060':upper=ratio[0]
|
||||
if not found:break
|
||||
self.update(phase=f'{context} Kontext · Grundverteilungen',current_context=context)
|
||||
primary_success=self.test_candidate(context,'5080',[primary],'none',[],'full')
|
||||
dual_success=[]
|
||||
for ratio in GPU_SPLITS:
|
||||
rows=self.test_candidate(context,'5080 + 3060',[primary,secondary],'layer',ratio,'full')
|
||||
if rows:dual_success.append(ratio)
|
||||
if not primary_success and not dual_success:
|
||||
self.test_candidate(context,'5080 + 3060 + CPU',[primary,secondary],'layer',[85,15],'auto')
|
||||
self.update(completed_contexts=self.job.get('completed_contexts',[])+[context])
|
||||
if dual_success and not primary_success:fine_search.append((context,dual_success[0]))
|
||||
# Run one-layer refinement only after every requested context has its
|
||||
# coarse 5080/3060 splits, so tuning cannot starve later context levels.
|
||||
for context,ratio in reversed(fine_search):
|
||||
try:
|
||||
entry=self.worker.catalog.entry(self.job['model_id'])
|
||||
total=model_layers(self.worker.catalog.root/entry['id']/('model'+Path(entry['file']).suffix),self.job['mtp'])
|
||||
except (OSError,ValueError,TypeError) as exc:
|
||||
self.update(fine_search_note='Layer-Feinsuche nicht verfügbar: '+str(exc));continue
|
||||
for layers,finer in fine_splits(total,ratio,100):
|
||||
result=self.test_candidate(context,'5080 + 3060 · Layer-Feinsuche',[primary,secondary],'layer',finer,'full',layers)
|
||||
if not result:break
|
||||
self.update(state='complete',phase='Testreihe abgeschlossen · Ergebnisse sind keine Garantie für beliebige Last',finished_at=time.time())
|
||||
finally:self.worker.stop()
|
||||
except TestBudget:self.update(state='complete',phase='Testbudget erreicht · Teilresultate verfügbar',finished_at=time.time())
|
||||
except TestBudget as exc:self.update(state='partial',phase=f'{exc} · Testreihe unvollständig, Teilresultate verfügbar',finished_at=time.time())
|
||||
except Exception as exc:self.update(state='cancelled' if self.cancel.is_set() else 'failed',phase='Abgebrochen' if self.cancel.is_set() else 'Test beendet',error=None if self.cancel.is_set() else str(exc),finished_at=time.time())
|
||||
def save(self,data):
|
||||
with self.lock:
|
||||
|
||||
Reference in New Issue
Block a user