394 lines
34 KiB
Python
394 lines
34 KiB
Python
"""Encrypted configuration export and sequential, inspectable recovery jobs.
|
|
No chats, prompts, media, logs, model weights or container layers are read into an export.
|
|
"""
|
|
import base64,copy,hashlib,json,os,re,secrets,shutil,sqlite3,tempfile,threading,time,zlib
|
|
from pathlib import Path,PurePosixPath
|
|
from auth import atomic_write,validate_record
|
|
from backup_codec import seal,open_backup
|
|
from catalog import repo_id,file_role,KINDS
|
|
from profiles import SCHEMAS,CHAT_GPU_DEFAULTS,chat_parameters
|
|
SETTINGS=('profiles.json','endpoint.json','audio-policy.json','video/comfy-selection.json','models/huggingface.json','video/comfy-client-token','video/original-work/settings.json')
|
|
MAX_HISTORY=40*1024**2
|
|
VERSION=1
|
|
# Exact byte hash verified against official Hub metadata, not a filename heuristic.
|
|
LOCAL_SOURCES={'83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c2395f5c2d53':dict(repo='unsloth/Qwen3.8-27B-GGUF',revision='4ca720788d1e01f1bff70c033e0d0028fd02e502',file='mmproj-BF16.gguf')}
|
|
|
|
def validate(doc):
|
|
if not isinstance(doc,dict) or doc.get('format')!='athena-deck' or doc.get('version')!=VERSION:raise ValueError('Backup-Version nicht unterstützt.')
|
|
if set(doc)-{'format','version','created_at','settings','models','credentials','theme','network','services','runtimes','warnings','history','runtime_versions'}:raise ValueError('Unbekannte Backup-Felder.')
|
|
if not isinstance(doc.get('warnings',[]),list) or len(doc.get('warnings',[]))>100 or any(not isinstance(w,str) or len(w)>1024 for w in doc.get('warnings',[])):raise ValueError('Ungültige Backup-Hinweise.')
|
|
if 'history' in doc and (not isinstance(doc['history'],str) or len(doc['history'])>56*1024**2):raise ValueError('History im Backup zu groß.')
|
|
models=doc.get('models');profiles=doc.get('settings',{}).get('profiles.json',[])
|
|
if not isinstance(models,list) or len(models)>4096 or not isinstance(profiles,list) or len(profiles)>1000:raise ValueError('Ungültiges Modell-/Profilverzeichnis.')
|
|
ids=set()
|
|
for m in models:
|
|
repo_id(m.get('repo'));f=m.get('file','');parts=PurePosixPath(f).parts
|
|
if not isinstance(f,str) or not parts or f.startswith('/') or '..' in parts or '\\' in f or len(f)>1024 or any(ord(c)<32 for c in f):raise ValueError('Unsicherer Modell-Dateiname.')
|
|
local=m.get('source')=='local'
|
|
if not re.fullmatch('[a-f0-9]{64}' if local else '[a-f0-9]{40}',m.get('revision','')) or m.get('kind') not in KINDS or type(m.get('size')) is not int or not 0<m['size']<4*1024**4:raise ValueError('Ungültige Modellquelle.')
|
|
ident=hashlib.sha256((m['repo']+m['revision']+f).encode()).hexdigest()
|
|
if (not local and m.get('id')!=ident) or not re.fullmatch('[a-f0-9]{64}',m.get('id','')) or m['id'] in ids or not re.fullmatch('[a-f0-9]{64}',m.get('sha256','')):raise ValueError('Ungültige Modell-ID oder Prüfsumme.')
|
|
if local:
|
|
if not m['repo'].startswith('local/') or m['revision']!=m['sha256']:raise ValueError('Ungültige lokale Modellherkunft.')
|
|
source=m.get('download_source')
|
|
if source is not None:
|
|
if not isinstance(source,dict) or set(source)!={'repo','revision','file'}:raise ValueError('Ungültige Ersatzquelle.')
|
|
repo_id(source['repo'])
|
|
if not re.fullmatch('[a-f0-9]{40}',source['revision']) or source['file'].startswith('/') or '..' in PurePosixPath(source['file']).parts or '\\' in source['file'] or any(ord(c)<32 for c in source['file']):raise ValueError('Ungültige Ersatzquelle.')
|
|
ids.add(m['id'])
|
|
pids=set();names=set()
|
|
for p in profiles:
|
|
if not isinstance(p,dict) or not re.fullmatch('[a-f0-9]{32}',p.get('id','')) or p['id'] in pids or p.get('name') in names:raise ValueError('Ungültige oder doppelte Profil-ID.')
|
|
name=p.get('name','');kind=p.get('kind');params=p.get('parameters')
|
|
if not isinstance(name,str) or not 2<=len(name)<=64 or not re.fullmatch(r'[\w .-]+',name) or not name[0].isalnum() or not name[-1].isalnum() or kind not in SCHEMAS or p.get('model_id') not in ids:raise ValueError('Ungültiges Profil oder Modellreferenz.')
|
|
if type(p.get('revision')) is not int or not 1<=p['revision']<=1000000:raise ValueError('Ungültige Profilrevision.')
|
|
if not isinstance(params,dict):raise ValueError('Ungültige Profilparameter.')
|
|
params=chat_parameters(params) if kind=='chat' else params
|
|
if set(params)!=(set(SCHEMAS[kind])|(set(CHAT_GPU_DEFAULTS) if kind=='chat' else set(params)&{'video_device','text_encoder_device'} if kind=='video' else set())):raise ValueError('Unbekannte Profilparameter.')
|
|
for key,(lo,hi,_) in SCHEMAS[kind].items():
|
|
value=params[key]
|
|
if isinstance(value,bool) or not isinstance(value,(int,float)) or not lo<=value<=hi or key not in ('temperature','top_p','guidance','speed','mtp_min_p') and type(value) is not int:raise ValueError('Ungültiger Profilparameter: '+key)
|
|
if kind=='chat' and (params['ubatch']>params['batch'] or params.get('vision_projector') and params['vision_projector'] not in ids):raise ValueError('Ungültige Chat-Zuordnung.')
|
|
if kind in ('image','video') and (params['width']%64 or params['height']%64):raise ValueError('Ungültige Bildgröße.')
|
|
components=p.get('components',{})
|
|
if not isinstance(components,dict) or any((k not in ('text_encoder','vae','video_vae','audio_vae','spatial_upsampler','model_config','generation_config','tokenizer','vae_config','manual_text_encoder','manual_vae','manual_tokenizer','manual_configuration','manual_projector','manual_auxiliary') and not re.fullmatch(r'runtime_file_[0-9]{1,2}',k)) or v not in ids for k,v in components.items()):raise ValueError('Ungültige Komponentenreferenz.')
|
|
pids.add(p['id']);names.add(name)
|
|
settings=doc.get('settings')
|
|
if not isinstance(settings,dict) or set(settings)-set(SETTINGS):raise ValueError('Unbekannte Einstellungsdateien.')
|
|
ep=settings.get('endpoint.json',{})
|
|
if not isinstance(ep,dict) or set(ep)-{'port','enabled_profiles','autostart'} or type(ep.get('port',8120)) is not int or not 1024<=ep.get('port',8120)<=65535 or type(ep.get('autostart',False)) is not bool:raise ValueError('Ungültige Endpunkt-Einstellungen.')
|
|
enabled=ep.get('enabled_profiles',[])
|
|
if not isinstance(enabled,list) or any(p not in pids for p in enabled) or len(set(enabled))!=len(enabled):raise ValueError('Unbekannte Profilfreigabe.')
|
|
if sum(p['kind']=='image' and p['id'] in enabled for p in profiles)>1:raise ValueError('Nur ein Bildprofil darf freigegeben sein.')
|
|
policy=settings.get('audio-policy.json',{})
|
|
if not isinstance(policy,dict) or set(policy)-{'tts','stt'} or any(not isinstance(v,dict) or set(v)!={'mode','profile_id'} or v['mode'] not in ('auto','warm','per_request') or v['profile_id'] is not None and v['profile_id'] not in pids for v in policy.values()):raise ValueError('Ungültige Audio-Einstellung.')
|
|
selection=settings.get('video/comfy-selection.json',{})
|
|
if isinstance(selection,dict) and selection.get('runtime','comfy') not in ('comfy','original'):raise ValueError('Unbekannte Video-Laufzeit.')
|
|
if not isinstance(selection,dict) or set(selection)-{'model_id','runtime'} or selection.get('model_id') and selection['model_id'] not in ids:raise ValueError('Ungültige Video-Auswahl.')
|
|
if 'models/huggingface.json' in settings:
|
|
hf=settings['models/huggingface.json']
|
|
if not isinstance(hf,dict) or set(hf)!={'token'} or hf['token'] is not None and (not isinstance(hf['token'],str) or not re.fullmatch(r'hf_[A-Za-z0-9]{10,252}',hf['token'])):raise ValueError('Ungültiger Hugging-Face-Zugang.')
|
|
if 'video/comfy-client-token' in settings and not re.fullmatch(r'[A-Za-z0-9_-]{32,256}',settings['video/comfy-client-token'].strip()):raise ValueError('Ungültiger Diensttoken.')
|
|
if 'video/original-work/settings.json' in settings and (not isinstance(settings['video/original-work/settings.json'],dict) or len(json.dumps(settings['video/original-work/settings.json']))>65536):raise ValueError('Ungültige LTX-Einstellungen.')
|
|
validate_record(doc.get('credentials'))
|
|
runtimes=doc.get('runtimes')
|
|
if not isinstance(runtimes,dict) or set(runtimes)-{'llama','llama_builds','image','tts','enhancers','swarm_nodes','ltx_original','audio_cpp','separator'} or not {'llama','llama_builds','image','tts','enhancers','swarm_nodes'}<=set(runtimes):raise ValueError('Ungültiges Laufzeitrezept.')
|
|
if 'separator' in runtimes:
|
|
from separator_runtime import MODELS
|
|
value=runtimes['separator']
|
|
if not isinstance(value,dict) or set(value)!={'installed','models'} or type(value['installed']) is not bool or not isinstance(value['models'],list) or len(value['models'])>len(MODELS) or any(not isinstance(m,str) or m not in MODELS for m in value['models']) or len(set(value['models']))!=len(value['models']):raise ValueError('Ungültiges Audio-Separator-Rezept.')
|
|
if 'audio_cpp' in runtimes and type(runtimes['audio_cpp']) is not bool:raise ValueError('Ungültige audio.cpp-Laufzeit.')
|
|
if 'ltx_original' in runtimes and type(runtimes['ltx_original']) is not bool:raise ValueError('Ungültige LTX-Laufzeit.')
|
|
if not isinstance(runtimes['llama_builds'],list) or len(runtimes['llama_builds'])>50 or any(not isinstance(b,dict) or set(b)!={'commit','backend'} or not re.fullmatch('[a-f0-9]{40}',b.get('commit','')) or b.get('backend') not in ('CUDA','CPU') for b in runtimes['llama_builds']):raise ValueError('Ungültige Buildrezepte.')
|
|
if runtimes['llama'] is not None:
|
|
item=runtimes['llama']
|
|
if not isinstance(item,dict) or not re.fullmatch('[a-f0-9]{40}',item.get('commit','')) or item.get('backend') not in ('CUDA','CPU'):raise ValueError('Ungültige llama.cpp-Quelle.')
|
|
if any(type(runtimes[k]) is not bool for k in ('image','tts','swarm_nodes')) or not isinstance(runtimes['enhancers'],dict) or set(runtimes['enhancers'])-{'t2i','i2i'}:raise ValueError('Ungültiges Laufzeitrezept.')
|
|
for revision in runtimes['enhancers'].values():
|
|
if not isinstance(revision,str) or not re.fullmatch('[a-f0-9]{40}',revision):raise ValueError('Ungültige Aufwerterversion.')
|
|
network=doc.get('network')
|
|
if network is not None:
|
|
if not isinstance(network,dict) or set(network)!={'config','enabled','mode'} or type(network['enabled']) is not bool or network['mode'] not in ('lan','both','tunnel'):raise ValueError('Ungültiger Netzwerkstand.')
|
|
if network['config'] is not None:
|
|
from network.config import parse_config
|
|
parse_config(network['config'])
|
|
services=doc.get('services')
|
|
if not isinstance(services,list) or len(services)>50:raise ValueError('Ungültiger Containerbestand.')
|
|
# Validation is shared with the root helper, which independently revalidates before Docker calls.
|
|
from deploy.docker_backup import validate as validate_container
|
|
seen=set()
|
|
for c in services:
|
|
validate_container(c)
|
|
if c['name'] in seen:raise ValueError('Doppelte Containerdefinition.')
|
|
seen.add(c['name'])
|
|
if doc.get('theme','deck') not in ('deck','athena-classic'):raise ValueError('Unbekanntes Theme.')
|
|
return doc
|
|
|
|
class Backup:
|
|
def __init__(self,server):
|
|
self.server=server;self.root=server.catalog.root.parent;self.directory=self.root/'recovery';self.directory.mkdir(parents=True,exist_ok=True,mode=0o700)
|
|
self.lock=threading.RLock();self.cancel=threading.Event();self.plan=None;self.job=None
|
|
path=self.directory/'job.json'
|
|
if path.exists():
|
|
self.job=json.loads(path.read_text())
|
|
if self.job.get('state')=='running':self.job.update(state='interrupted',phase='Restore durch Neustart unterbrochen. Backup erneut importieren; vorhandene Dateien werden geprüft und wiederverwendet.');self._save()
|
|
def busy(self):return bool(self.job and self.job['state']=='running')
|
|
def _save(self):atomic_write(self.directory/'job.json',self.job)
|
|
def status(self):
|
|
with self.lock:return {'job':copy.deepcopy(self.job),'preview':copy.deepcopy(self.plan['summary']) if self.plan else None,'checkpoints':[p.name for p in sorted(self.directory.glob('before-*.adbackup'),reverse=True)]}
|
|
def snapshot(self,theme='deck',include_history=True,_during_restore=False):
|
|
s=self.server
|
|
if self.busy() and not _during_restore:raise ValueError('Restore läuft. Export erst danach.')
|
|
settings={}
|
|
# Explicit allowlist: no recursive backup of process work directories.
|
|
with s.profiles.lock,s.endpoint.lock,s.catalog.lock:
|
|
for name in SETTINGS:
|
|
p=self.root/name
|
|
if p.is_file():settings[name]=p.read_text().strip() if name.endswith('comfy-client-token') else json.loads(p.read_text())
|
|
models=[]
|
|
for m in s.catalog.status()['entries']:
|
|
item={k:m[k] for k in ('id','repo','revision','file','size','sha256','kind')}
|
|
if m['repo'].startswith('local/'):
|
|
item['source']='local';item['download_source']=m.get('download_source') or LOCAL_SOURCES.get(m['sha256'])
|
|
models.append(item)
|
|
state=s.runtime.status();active=next((b for b in state['builds'] if b['id']==state['active']),None)
|
|
llama={'commit':active['commit'],'backend':active['backend']} if active else None
|
|
warnings=[];services=[]
|
|
try:services=s.docker.call('backup-export')['services']
|
|
except (OSError,ValueError):
|
|
if s.docker.status().get('services'):raise ValueError('Container-Konfigurationen können ohne aktualisierten Systemhelfer nicht gesichert werden.')
|
|
warnings.append('Docker-Systemhelfer nicht verfügbar; keine Zusatzdienste gesichert.')
|
|
network=None
|
|
if not s.network.disabled:
|
|
network=s.network.call('backup-export')
|
|
else:warnings.append('Deck-Netzwerkmodul deaktiviert; fremder WireGuard-Gateway wird nicht gesichert.')
|
|
document=dict(format='athena-deck',version=VERSION,created_at=time.time(),settings=settings,models=models,credentials=s.credentials.read(),theme=theme,network=network,services=services,runtime_versions=dict(image=dict(comfy=s.image_runtime.status()['comfy_revision'],gguf=s.image_runtime.status()['gguf_revision']),tts=s.tts_runtime.status()['revision']),runtimes=dict(audio_cpp=bool(getattr(s,'audio_cpp_runtime',None) and s.audio_cpp_runtime.status()['installed']),ltx_original=bool(getattr(s,'ltx_original_runtime',None) and s.ltx_original_runtime.status()['installed']),llama=llama,llama_builds=[dict(commit=b['commit'],backend=b['backend']) for b in state['builds']],image=s.image_runtime.status()['installed'],tts=s.tts_runtime.status()['installed'],enhancers={task:data['revision'] for task in ('t2i','i2i') if (data:=s.prompt_enhancer.installed(task))},swarm_nodes=(self.root/'video/swarm-comfy-nodes').is_dir()),warnings=warnings)
|
|
if getattr(s,'separator_runtime',None):
|
|
separator=s.separator_runtime.status();document['runtimes']['separator']=dict(installed=separator['installed'],models=[m['id'] for m in separator['models'] if m['installed']]);document['runtime_versions']['separator']=separator['version']
|
|
if document['runtimes']['audio_cpp']:
|
|
from audio_cpp_runtime import REVISION
|
|
document['runtime_versions']['audio_cpp']=REVISION
|
|
if document['runtimes']['ltx_original']:
|
|
from ltx_original_runtime import DESKTOP_REV,LTX_REV
|
|
document['runtime_versions']['ltx_original']=dict(desktop=DESKTOP_REV,inference=LTX_REV)
|
|
if include_history:
|
|
with tempfile.TemporaryDirectory(dir=self.directory) as work:
|
|
out=Path(work)/'history.sqlite3'
|
|
with s.dashboard.history._lock,sqlite3.connect(out) as dest:s.dashboard.history._db.backup(dest)
|
|
if out.stat().st_size>MAX_HISTORY:raise ValueError('Dashboard-History größer als 40 MiB. History separat sichern oder Export ohne History wählen.')
|
|
document['history']=base64.b64encode(out.read_bytes()).decode()
|
|
return document
|
|
def export(self,phrase,theme='deck',history=True):return seal(validate(self.snapshot(theme,history)),phrase)
|
|
def inspect(self,raw,phrase):
|
|
doc=validate(open_backup(raw,phrase));s=self.server;blockers=[];warnings=list(doc.get('warnings',[]));existing={m['id']:m for m in s.catalog.status()['entries']}
|
|
if 'history' in doc:self._restore_history(doc['history'],validate_only=True)
|
|
missing=[m for m in doc['models'] if m['id'] not in existing];need=sum(m['size'] for m in missing)
|
|
if missing and need+10*1024**3>shutil.disk_usage(self.root).free:blockers.append('Zu wenig Platz für fehlende Gewichte einschließlich 10 GiB Reserve.')
|
|
for m in doc['models']:
|
|
if m.get('source')=='local' and not m.get('download_source'):
|
|
warnings.append(m['file']+': lokale Datei ohne nachladbare Quelle; separat sichern.')
|
|
if m['id'] not in existing:blockers.append(m['file']+': lokale Gewichtedatei fehlt und hat keine Downloadquelle.')
|
|
if m['id'] in existing and existing[m['id']].get('sha256')!=m['sha256']:blockers.append('Vorhandene Modelldatei hat abweichende Prüfsumme: '+m['file'])
|
|
if doc['runtimes']['llama']:
|
|
prereq=s.runtime.prerequisites();backend=doc['runtimes']['llama']['backend']
|
|
if not prereq['cuda_ready' if backend=='CUDA' else 'cpu_ready']:blockers.append('llama.cpp-Buildwerkzeuge fehlen; Laufzeit-Voraussetzungen installieren.')
|
|
if doc['network'] and doc['network'].get('config'):
|
|
if s.network.disabled:blockers.append('Backup enthält WireGuard: Deck-Netzwerkmodul muss zuerst installiert/angebunden werden. Bestehender Gateway wird nicht übernommen.')
|
|
elif s.network.status().get('enabled') or s.network.status().get('mode')!='lan':blockers.append('Deck-Tunnel vor Netzwerk-Restore im bestätigten LAN-Modus deaktivieren.')
|
|
if doc['network']['mode']!='lan':warnings.append('Tunnelzugang wird mit Rückfallzeit aktiviert und muss über den Tunnel bestätigt werden.')
|
|
versions=doc.get('runtime_versions',{})
|
|
if versions.get('separator'):
|
|
from separator_runtime import VERSION as SEPARATOR_VERSION
|
|
if versions['separator']!=SEPARATOR_VERSION:blockers.append('Backup verlangt eine andere Audio-Separator-Version.')
|
|
if versions.get('audio_cpp'):
|
|
from audio_cpp_runtime import REVISION
|
|
if versions['audio_cpp']!=REVISION:blockers.append('Backup verlangt eine andere audio.cpp-Version. Passende Deck-Version verwenden.')
|
|
if versions.get('ltx_original'):
|
|
from ltx_original_runtime import DESKTOP_REV,LTX_REV
|
|
if versions['ltx_original']!=dict(desktop=DESKTOP_REV,inference=LTX_REV):blockers.append('Backup verlangt eine andere originale LTX-Version. Passende Deck-Version verwenden.')
|
|
if versions and {k:v for k,v in versions.items() if k not in ('ltx_original','audio_cpp','separator')}!=dict(image=dict(comfy=s.image_runtime.status()['comfy_revision'],gguf=s.image_runtime.status()['gguf_revision']),tts=s.tts_runtime.status()['revision']):blockers.append('Backup verlangt andere Bild-/TTS-Laufzeitversionen. Passende Deck-Version oder geprüftes Runtime-Upgrade verwenden.')
|
|
for c in doc['services']:
|
|
if not c.get('registry') and c.get('build_recipe')!='swarm-ui':warnings.append(c['name']+': Image nur lokal vorhanden; Registry-/Buildquelle fehlt.')
|
|
ports=s.endpoint.allowed_ports
|
|
if ports and doc['settings'].get('endpoint.json',{}).get('port',8120) not in ports:blockers.append('API-Port aus Backup ist nicht veröffentlicht. Installer zuerst für diesen Port konfigurieren.')
|
|
ident=secrets.token_hex(16);summary=dict(id=ident,created_at=doc.get('created_at'),models=[dict(id=m['id'],name=m['file'],bytes=m['size'],present=m['id'] in existing) for m in doc['models']],profiles=[p['name'] for p in doc['settings'].get('profiles.json',[])],services=[dict(name=c['name'],image=c['image'],registry=c.get('registry'),build_recipe=c.get('build_recipe'),restorable=bool(c.get('registry') or c.get('build_recipe')),running=c['running']) for c in doc['services']],download_bytes=need,runtimes=doc['runtimes'],wireguard=bool(doc['network'] and doc['network']['config']),warnings=warnings,blockers=blockers,theme=doc.get('theme','deck'))
|
|
with self.lock:
|
|
if self.busy():raise ValueError('Restore läuft bereits.')
|
|
self.plan={'document':doc,'summary':summary,'expires':time.monotonic()+1800}
|
|
return summary
|
|
def inspect_checkpoint(self,name):
|
|
if not isinstance(name,str) or not re.fullmatch(r'before-[a-f0-9]{32}\.adbackup',name):raise ValueError('Ungültiger Rückfallstand.')
|
|
p=self.directory/name;key=p.with_suffix('.key')
|
|
if p.is_symlink() or key.is_symlink() or not p.is_file() or not key.is_file():raise ValueError('Rückfallstand nicht verfügbar.')
|
|
return self.inspect(p.read_bytes(),key.read_text())
|
|
def start(self,data):
|
|
if not isinstance(data,dict) or set(data)!={'id','services','confirm','restore_credentials'} or data['confirm'] is not True or type(data['restore_credentials']) is not bool or not isinstance(data['services'],list):raise ValueError('Restore-Prüfung und ausdrückliche Bestätigung erforderlich.')
|
|
with self.lock:
|
|
if self.busy():raise ValueError('Restore läuft bereits.')
|
|
if not self.plan or self.plan['expires']<time.monotonic() or self.plan['summary']['id']!=data['id']:raise ValueError('Importprüfung abgelaufen. Backup neu auswählen.')
|
|
if self.plan['summary']['blockers']:raise ValueError('Zuerst die Hindernisse aus der Importprüfung beheben.')
|
|
doc=self.plan['document'];services={c['name']:c for c in doc['services']}
|
|
if len(set(data['services']))!=len(data['services']) or any(name not in services for name in data['services']):raise ValueError('Unbekannter Zusatzdienst.')
|
|
if any(not services[name].get('registry') and not services[name].get('build_recipe') for name in data['services']):raise ValueError('Gewählter Dienst hat keine Wiederherstellungsquelle.')
|
|
s=self.server
|
|
if s.catalog.pending or s.catalog.job and s.catalog.job['state']=='downloading' or s.runtime.busy or any((getattr(o,'job',None) or {}).get('state')=='running' for o in (s.image_runtime,s.tts_runtime,s.ltx_original_runtime,getattr(s,'audio_cpp_runtime',None),getattr(s,'music',None),getattr(s,'separator_runtime',None),getattr(s,'separator_tests',None),s.prompt_enhancer,s.auto_tests,s.chat_tests,s.image_tests,s.tts_tests,s.stt)):raise ValueError('Zuerst laufende Downloads, Builds und Tests abschließen oder abbrechen.')
|
|
self.cancel.clear();self.job=dict(id=secrets.token_hex(16),state='running',phase='Wiederherstellung vorbereiten',completed=0,total=len(doc['models'])+len(data['services'])+5,items=[],started_at=time.time(),theme=doc.get('theme','deck'))
|
|
self._save();threading.Thread(target=self._restore,args=(copy.deepcopy(doc),data['services'],data['restore_credentials']),daemon=True).start();self.plan=None
|
|
return self.status()
|
|
def _phase(self,text):
|
|
with self.lock:self.job['phase']=text;self._save()
|
|
def _item(self,name,state,message):
|
|
with self.lock:self.job['items'].append(dict(name=name,state=state,message=message));self.job['completed']+=1;self._save()
|
|
def _check(self):
|
|
if self.cancel.is_set():raise InterruptedError()
|
|
def _wait(self,status,get_job=lambda x:x.get('job')):
|
|
while True:
|
|
self._check();value=status();job=get_job(value)
|
|
if not job or job.get('state') not in ('running','queued','downloading'):break
|
|
self._phase(job.get('phase') or ('Gewichte herunterladen: '+job.get('file','')) if job.get('file') else job.get('phase','Dateien laden / prüfen'))
|
|
with self.lock:self.job['current']={k:job[k] for k in ('bytes','total','bytes_per_second','eta_seconds') if k in job};self._save()
|
|
self.cancel.wait(1)
|
|
if job and job.get('state') not in ('complete',):raise ValueError('Download oder Laufzeitinstallation fehlgeschlagen. Zugang, Speicher und Voraussetzungen prüfen.')
|
|
with self.lock:self.job.pop('current',None);self._save()
|
|
return value
|
|
def _runtimes(self,doc):
|
|
s=self.server;r=doc['runtimes']
|
|
if r['image'] and not s.image_runtime.status()['installed']:
|
|
self._phase('Vorhandene Bild-/Videolaufzeit prüfen oder installieren');s.image_runtime.start();self._wait(s.image_runtime.status)
|
|
if r['tts'] and not s.tts_runtime.status()['installed']:
|
|
self._phase('TTS-Laufzeit und Zusatzdateien installieren');s.tts_runtime.start();self._wait(s.tts_runtime.status)
|
|
if r.get('audio_cpp') and not s.audio_cpp_runtime.status()['installed']:
|
|
self._phase('audio.cpp-Laufzeit installieren');s.audio_cpp_runtime.start();self._wait(s.audio_cpp_runtime.status)
|
|
if r.get('ltx_original') and not s.ltx_original_runtime.status()['installed']:
|
|
self._phase('Originale LTX-Laufzeit installieren');s.ltx_original_runtime.start();self._wait(s.ltx_original_runtime.status)
|
|
if r.get('separator',{}).get('installed'):
|
|
if not s.separator_runtime.status()['installed']:
|
|
self._phase('Audio Separator installieren');s.separator_runtime.start();self._wait(s.separator_runtime.status)
|
|
for name in r['separator']['models']:
|
|
if not s.separator_runtime.model_ready(name):
|
|
self._phase('Audio-Separator-Modellpaket laden');s.separator_runtime.download(name);self._wait(s.separator_runtime.status)
|
|
targets=r['llama_builds']+([r['llama']] if r['llama'] and r['llama'] not in r['llama_builds'] else [])
|
|
for target in targets:
|
|
state=s.runtime.status();matching=next((b for b in state['builds'] if b['commit']==target['commit'] and b['backend']==target['backend']),None)
|
|
if not matching:
|
|
self._phase('llama.cpp-Version aus Backup bauen');s.runtime.start(target['commit'],target['backend'],1);state=self._wait(s.runtime.status);matching=state['builds'][-1]
|
|
if target==r['llama']:s.runtime.activate(matching['id'])
|
|
for task,revision in r['enhancers'].items():
|
|
if not s.prompt_enhancer.installed(task):
|
|
s.prompt_enhancer.install(task,revision=revision);self._wait(s.prompt_enhancer.status)
|
|
self._item('Laufzeiten','complete','Vorhandene Installationen wiederverwendet; fehlende Laufzeiten eingerichtet.')
|
|
def _restore(self,doc,selected,restore_credentials):
|
|
s=self.server;errors=[];rollback=None;before=None
|
|
try:
|
|
# Preserve non-reproducible settings before touching anything; never overwrite old rollback files.
|
|
self._phase('Lokalen Rückfallstand sichern')
|
|
rollback=self.directory/('before-'+self.job['id']+'.adbackup')
|
|
before=self.snapshot_for_rollback()
|
|
key=secrets.token_urlsafe(32);rollback.write_bytes(seal(before,key));rollback.chmod(0o600)
|
|
# Recovery key stays on this server in a separate protected file, never in status responses.
|
|
(rollback.with_suffix('.key')).write_text(key);rollback.with_suffix('.key').chmod(0o600)
|
|
s.endpoint.close();s.video.close();s.tts_tests.stop();s.stt.stop()
|
|
until=time.monotonic()+30
|
|
while s.endpoint.inflight and time.monotonic()<until:time.sleep(.1)
|
|
if s.endpoint.inflight:raise ValueError('API-Aufträge wurden nicht rechtzeitig beendet. Restore erneut versuchen.')
|
|
with s.endpoint.lock:s.endpoint.http=None;s.endpoint.thread=None;s.endpoint.state='stopped'
|
|
with s.scheduler.cv:s.scheduler.gpu_mode='restoring';s.scheduler.cv.notify_all()
|
|
self._item('Rückfallstand','complete','Vorherige Einstellungen geschützt gesichert; nur Deck-eigene Worker beendet.')
|
|
hf=doc['settings'].get('models/huggingface.json')
|
|
if hf:atomic_write(self.root/'models/huggingface.json',hf)
|
|
self._runtimes(doc)
|
|
for m in doc['models']:
|
|
self._check();self._phase('Gewichte: '+m['file'])
|
|
try:
|
|
entry=s.catalog.entry(m['id']);path=s.catalog.root/m['id']/('model'+PurePosixPath(m['file']).suffix)
|
|
digest=hashlib.sha256()
|
|
with path.open('rb') as stream:
|
|
for block in iter(lambda:stream.read(4*1024**2),b''):self._check();digest.update(block)
|
|
if digest.hexdigest()!=m['sha256']:raise ValueError('Vorhandene Datei stimmt nicht mit Backup-Prüfsumme überein.')
|
|
self._item(m['file'],'reused','Vorhandene Datei per SHA-256 geprüft.')
|
|
except ValueError:
|
|
if (s.catalog.root/m['id']/'entry.json').exists():raise
|
|
source=m.get('download_source') or m
|
|
s.catalog.start(source['repo'],source['file'],source['revision'],m['kind']);self._wait(s.catalog.status)
|
|
source_id=hashlib.sha256((source['repo']+source['revision']+source['file']).encode()).hexdigest()
|
|
entry=s.catalog.entry(source_id)
|
|
if entry['sha256']!=m['sha256']:raise ValueError('Geladene Datei stimmt nicht mit Backup-Prüfsumme überein.')
|
|
if source_id!=m['id']:
|
|
target=s.catalog.root/m['id']
|
|
if target.exists():
|
|
if any(target.iterdir()):raise ValueError('Lokaler Modellordner ist anderweitig belegt.')
|
|
target.rmdir()
|
|
(s.catalog.root/source_id).rename(target)
|
|
old=target/('model'+PurePosixPath(source['file']).suffix);new=target/('model'+PurePosixPath(m['file']).suffix)
|
|
if old!=new:old.rename(new)
|
|
entry.update({k:m[k] for k in ('repo','revision','file','kind')});entry['download_source']=m['download_source'];entry.pop('id',None);atomic_write(target/'entry.json',entry)
|
|
self._item(m['file'],'complete','Feste Quellversion geladen und SHA-256 geprüft.')
|
|
for c in doc['services']:
|
|
self._check()
|
|
if c['name'] not in selected:self._item(c['name'],'skipped','Vom Restore ausgeschlossen.');continue
|
|
self._phase('Zusatzdienst: '+c['name'])
|
|
try:result=s.docker.call('backup-restore',values={'container':c});self._item(c['name'],result['state'],result['message'])
|
|
except (OSError,ValueError):errors.append(c['name']);self._item(c['name'],'failed','Container nicht wiederhergestellt; Registry, Quelle, Ports oder Systemhelfer prüfen.')
|
|
if doc['runtimes']['swarm_nodes'] and 'athena-swarm-ui' in selected:
|
|
self._phase('Swarm-Abhängigkeiten zur vorhandenen Video-Laufzeit ergänzen')
|
|
from image_runtime import ImageRuntime
|
|
python,_=s.image_runtime.paths()
|
|
import subprocess
|
|
subprocess.run([str(python),'-m','pip','install','--upgrade','--no-cache-dir','--no-deps','--target',str(self.root/'video/swarm-python'),'opencv-python-headless==4.12.0.88','imageio-ffmpeg==0.6.0'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=True,timeout=300)
|
|
self._check();self._phase('Profile, Zuordnungen und Einstellungen anwenden')
|
|
self._apply_settings(doc)
|
|
if 'history' in doc:
|
|
self._restore_history(doc['history'])
|
|
self._item('Einstellungen','complete','Profile, API-Freigaben, Komponenten und Audio-Verhalten gesetzt.')
|
|
if doc['network'] and doc['network']['config']:
|
|
try:
|
|
result=s.network.call('backup-restore',{'backup':doc['network']})
|
|
if result.get('pending'):errors.append('WireGuard-Bestätigung')
|
|
self._item('WireGuard','pending' if result.get('pending') else 'complete','Deck-Konfiguration gesetzt; Handshake geprüft. Tunnelmodus gegebenenfalls im Netzwerkbereich bestätigen.')
|
|
except (OSError,ValueError):errors.append('WireGuard');self._item('WireGuard','failed','Netzwerk nicht vollständig wiederhergestellt. LAN-Zugang bleibt erhalten; Netzwerkbereich prüfen.')
|
|
else:self._item('WireGuard','skipped','Kein Deck-WireGuard im Backup; vorhandene Gateways bleiben unberührt.')
|
|
if restore_credentials:
|
|
current=s.credentials.read();s.credentials.write(doc['credentials'],current['revision'])
|
|
with s.auth_lock:
|
|
for session in s.sessions.values():session['revision']=doc['credentials']['password']['hash']
|
|
self._item('Zugang','complete','Oberflächenkennwort und API-Token-Prüfwert aus Backup wiederhergestellt.')
|
|
else:self._item('Zugang','skipped','Aktueller Oberflächenzugang und API-Token bleiben erhalten.')
|
|
with s.scheduler.cv:s.scheduler.gpu_mode='video' if s.video.foreign_blocked else 'llm';s.scheduler.cv.notify_all()
|
|
if s.endpoint.config.get('autostart'):
|
|
try:s.endpoint.start()
|
|
except (OSError,ValueError):errors.append('API-Endpunkt');self._item('API-Endpunkt','failed','Listener konnte nicht gestartet werden; Port prüfen.')
|
|
with self.lock:self.job.update(state='partial' if errors else 'complete',phase='Wiederherstellung mit offenen Punkten beendet.' if errors else 'Wiederherstellung abgeschlossen. Oberfläche ist bereit.',finished_at=time.time(),completed=self.job['total']);self._save()
|
|
except Exception as exc:
|
|
# Restore preexisting application settings; downloaded files/created services remain available.
|
|
if before is not None:
|
|
try:
|
|
self._apply_settings(before)
|
|
old=before['runtimes']['llama']
|
|
if old:
|
|
item=next((b for b in s.runtime.status()['builds'] if b['commit']==old['commit'] and b['backend']==old['backend']),None)
|
|
if item:s.runtime.activate(item['id'])
|
|
if s.endpoint.config.get('autostart'):s.endpoint.start()
|
|
except Exception:pass
|
|
|
|
with s.scheduler.cv:s.scheduler.gpu_mode='video' if s.video.foreign_blocked else 'llm';s.scheduler.cv.notify_all()
|
|
self._item('Restore','cancelled' if isinstance(exc,InterruptedError) else 'failed',str(exc) if isinstance(exc,ValueError) else 'Wiederherstellung unterbrochen. Rückfallstand vorhanden; Backup erneut prüfen.')
|
|
with self.lock:self.job.update(state='cancelled' if isinstance(exc,InterruptedError) else 'failed',phase='Restore nicht abgeschlossen. Rückfall auf vorherige Einstellungen versucht; geschützter Rückfallstand vorhanden.',finished_at=time.time());self._save()
|
|
def _apply_settings(self,doc):
|
|
s=self.server
|
|
for name in SETTINGS:
|
|
if name not in doc['settings']:(self.root/name).unlink(missing_ok=True)
|
|
for name,value in doc['settings'].items():
|
|
if name=='video/comfy-client-token':
|
|
path=self.root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_text(value+'\n');path.chmod(0o600)
|
|
else:atomic_write(self.root/name,value)
|
|
with s.profiles.lock:s.profiles.rows=copy.deepcopy(doc['settings'].get('profiles.json',[]))
|
|
with s.endpoint.lock:s.endpoint.config=copy.deepcopy(doc['settings'].get('endpoint.json',dict(port=8120,enabled_profiles=[],autostart=False)))
|
|
s.video.selection=copy.deepcopy(doc['settings'].get('video/comfy-selection.json',{}))
|
|
policy=doc['settings'].get('audio-policy.json',{k:{'mode':'auto','profile_id':None} for k in ('tts','stt')})
|
|
with s.audio_policy.lock:s.audio_policy.settings=policy
|
|
s.tts_tests.policy=policy.get('tts',{}).get('mode','auto');s.stt.policy=policy.get('stt',{}).get('mode','auto')
|
|
def snapshot_for_rollback(self):
|
|
return self.snapshot(include_history=False,_during_restore=True)
|
|
def _restore_history(self,data,validate_only=False):
|
|
raw=base64.b64decode(data,validate=True)
|
|
if len(raw)>MAX_HISTORY:raise ValueError('History im Backup zu groß.')
|
|
with tempfile.TemporaryDirectory(dir=self.directory) as work:
|
|
path=Path(work)/'history.sqlite3';path.write_bytes(raw)
|
|
with sqlite3.connect('file:'+str(path)+'?mode=ro',uri=True) as source:
|
|
if source.execute('PRAGMA quick_check').fetchone()[0]!='ok':raise ValueError('Ungültige Dashboard-History.')
|
|
expected={'meta','samples_raw','samples_hourly','slot_samples_raw','slot_samples_hourly','token_totals','token_hourly','model_events'}
|
|
if {r[0] for r in source.execute("SELECT name FROM sqlite_master WHERE type='table'")}!=expected or source.execute("SELECT COUNT(*) FROM sqlite_master WHERE type IN ('trigger','view')").fetchone()[0]:raise ValueError('Unbekanntes History-Schema.')
|
|
if not validate_only:
|
|
with self.server.dashboard.history._lock:source.backup(self.server.dashboard.history._db)
|
|
def stop(self):
|
|
with self.lock:
|
|
self.cancel.set()
|
|
if self.busy():
|
|
self.server.catalog.stop();self.server.runtime.stop();self.server.image_runtime.stop();self.server.tts_runtime.stop();self.server.ltx_original_runtime.stop();
|
|
if getattr(self.server,'separator_runtime',None):self.server.separator_runtime.stop()
|
|
if getattr(self.server,'separator_tests',None):self.server.separator_tests.stop()
|
|
if getattr(self.server,'music',None):self.server.music.stop()
|
|
if getattr(self.server,'audio_cpp_runtime',None):self.server.audio_cpp_runtime.stop()
|
|
self.server.prompt_enhancer.stop()
|
|
return {'cancellation_requested':True}
|