diff --git a/dist/urbm-2026.07.10.r006-x86_64-1.txz.sha256 b/dist/urbm-2026.07.10.r006-x86_64-1.txz.sha256 deleted file mode 100644 index 4246dcb..0000000 --- a/dist/urbm-2026.07.10.r006-x86_64-1.txz.sha256 +++ /dev/null @@ -1 +0,0 @@ -24660d70ea52abe59214ad8a56f31023cef090cb2fafeb557690870ed5790aa9 urbm-2026.07.10.r006-x86_64-1.txz diff --git a/dist/urbm-2026.07.10.r006-x86_64-1.txz b/dist/urbm-2026.07.10.r007-x86_64-1.txz similarity index 56% rename from dist/urbm-2026.07.10.r006-x86_64-1.txz rename to dist/urbm-2026.07.10.r007-x86_64-1.txz index 17c6480..2a7338d 100644 Binary files a/dist/urbm-2026.07.10.r006-x86_64-1.txz and b/dist/urbm-2026.07.10.r007-x86_64-1.txz differ diff --git a/dist/urbm-2026.07.10.r007-x86_64-1.txz.sha256 b/dist/urbm-2026.07.10.r007-x86_64-1.txz.sha256 new file mode 100644 index 0000000..1761141 --- /dev/null +++ b/dist/urbm-2026.07.10.r007-x86_64-1.txz.sha256 @@ -0,0 +1 @@ +3e3f832d98656655d591c05ad4454d8480d5cfcdf60c03a2a498a6498e206beb urbm-2026.07.10.r007-x86_64-1.txz diff --git a/dist/urbm.plg b/dist/urbm.plg index 8a22768..fabbe86 100644 --- a/dist/urbm.plg +++ b/dist/urbm.plg @@ -2,13 +2,17 @@ - + - + ]> +### 2026.07.10.r007 +- Serve snapshot browser folders as compact cached tree pages instead of sending the complete Restic file list to the browser. +- Prevent large snapshots with hundreds of thousands of entries from failing with generic 500 responses during snapshot browsing. + ### 2026.07.10.r006 - Stop logging high-frequency health, log, and activity polling requests unless they are slow or fail. - Log snapshot file-listing requests immediately when they start and show a larger daemon log tail in the Protokolle tab. diff --git a/internal/service/service.go b/internal/service/service.go index ae71e0f..5039e8a 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -10,6 +10,7 @@ import ( "log/slog" "os" "path/filepath" + "sort" "strings" "sync" "syscall" @@ -32,17 +33,49 @@ type SecretStore interface { } type Service struct { - store *store.Store - secrets SecretStore - restic *restic.Runner - rsync *rsync.Runner - mounts *platform.MountManager - workloads *platform.WorkloadManager - notifier *notify.Sender - log *slog.Logger - queue *queue.Queue - mu sync.RWMutex - config model.Config + store *store.Store + secrets SecretStore + restic *restic.Runner + rsync *rsync.Runner + mounts *platform.MountManager + workloads *platform.WorkloadManager + notifier *notify.Sender + log *slog.Logger + queue *queue.Queue + mu sync.RWMutex + config model.Config + snapshotMu sync.Mutex + snapshotTree map[string]snapshotTreeCache +} + +type SnapshotBrowserNode struct { + Path string `json:"path"` + Name string `json:"name"` + Type string `json:"type"` + Size int64 `json:"size,omitempty"` + HasChildren bool `json:"hasChildren"` +} + +type SnapshotBrowserPage struct { + Path string `json:"path"` + Items []SnapshotBrowserNode `json:"items"` + Total int `json:"total"` + Cached bool `json:"cached"` + Generated time.Time `json:"generated"` +} + +type snapshotTreeCache struct { + generated time.Time + total int + children map[string][]SnapshotBrowserNode +} + +type snapshotTreeNode struct { + path string + name string + nodeType string + size int64 + children map[string]*snapshotTreeNode } func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runner, mounts *platform.MountManager, workloads *platform.WorkloadManager, notifier *notify.Sender, log *slog.Logger) (*Service, error) { @@ -384,26 +417,162 @@ func (s *Service) RepositoryStats(ctx context.Context) []model.RepositoryStats { return result } -func (s *Service) SnapshotFiles(ctx context.Context, repoID, snapshot, path string) ([]map[string]any, error) { +func (s *Service) SnapshotFiles(ctx context.Context, repoID, snapshot, path string) (SnapshotBrowserPage, error) { started := time.Now() s.log.Info("snapshot file listing started", "repoID", repoID, "snapshot", snapshot, "path", path) repo, ok := s.repository(repoID) if !ok { - return nil, errors.New("validation: unknown repository") + return SnapshotBrowserPage{}, errors.New("validation: unknown repository") } + + normalizedPath := normalizeSnapshotPath(path) + key := snapshotCacheKey(repoID, snapshot) + if page, ok := s.snapshotBrowserPage(key, normalizedPath); ok { + s.log.Info("snapshot file listing loaded from cache", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "items", len(page.Items), "total", page.Total, "durationMs", time.Since(started).Milliseconds()) + return page, nil + } + mounted, err := s.mounts.Prepare(ctx, repo) if err != nil { - s.log.Warn("snapshot file listing failed", "repoID", repoID, "snapshot", snapshot, "path", path, "durationMs", time.Since(started).Milliseconds(), "error", err) - return nil, err + s.log.Warn("snapshot file listing failed", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "durationMs", time.Since(started).Milliseconds(), "error", err) + return SnapshotBrowserPage{}, err } defer s.mounts.Cleanup(context.Background(), mounted) - items, err := s.restic.List(ctx, mounted.Repository, snapshot, path) + + items, err := s.restic.List(ctx, mounted.Repository, snapshot, "") if err != nil { - s.log.Warn("snapshot file listing failed", "repoID", repoID, "snapshot", snapshot, "path", path, "durationMs", time.Since(started).Milliseconds(), "error", err) - return nil, err + s.log.Warn("snapshot file listing failed", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "durationMs", time.Since(started).Milliseconds(), "error", err) + return SnapshotBrowserPage{}, err } - s.log.Info("snapshot file listing loaded", "repoID", repoID, "snapshot", snapshot, "path", path, "items", len(items), "durationMs", time.Since(started).Milliseconds()) - return items, nil + cache := buildSnapshotTreeCache(items) + s.storeSnapshotTree(key, cache) + page, _ := s.snapshotBrowserPage(key, normalizedPath) + s.log.Info("snapshot file listing loaded", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "items", len(page.Items), "total", cache.total, "durationMs", time.Since(started).Milliseconds()) + return page, nil +} + +func snapshotCacheKey(repoID, snapshot string) string { + return repoID + "\x00" + snapshot +} + +func (s *Service) snapshotBrowserPage(key, path string) (SnapshotBrowserPage, bool) { + s.snapshotMu.Lock() + defer s.snapshotMu.Unlock() + cache, ok := s.snapshotTree[key] + if !ok || time.Since(cache.generated) > 30*time.Minute { + return SnapshotBrowserPage{}, false + } + items, ok := cache.children[path] + if !ok { + items = []SnapshotBrowserNode{} + } + return SnapshotBrowserPage{Path: path, Items: items, Total: cache.total, Cached: true, Generated: cache.generated}, true +} + +func (s *Service) storeSnapshotTree(key string, cache snapshotTreeCache) { + s.snapshotMu.Lock() + defer s.snapshotMu.Unlock() + if s.snapshotTree == nil { + s.snapshotTree = map[string]snapshotTreeCache{} + } + for cacheKey, value := range s.snapshotTree { + if time.Since(value.generated) > 30*time.Minute { + delete(s.snapshotTree, cacheKey) + } + } + s.snapshotTree[key] = cache +} + +func buildSnapshotTreeCache(items []map[string]any) snapshotTreeCache { + root := &snapshotTreeNode{path: "/", name: "/", nodeType: "dir", children: map[string]*snapshotTreeNode{}} + nodes := map[string]*snapshotTreeNode{"/": root} + ensure := func(path string, nodeType string, size int64) *snapshotTreeNode { + return root + } + ensure = func(path string, nodeType string, size int64) *snapshotTreeNode { + path = normalizeSnapshotPath(path) + if node, ok := nodes[path]; ok { + if nodeType != "" && nodeType != "dir" { + node.nodeType = nodeType + node.size = size + } + return node + } + parentPath := "/" + if path != "/" { + parentPath = filepath.Dir(path) + if parentPath == "." { + parentPath = "/" + } + } + parent := ensure(parentPath, "dir", 0) + node := &snapshotTreeNode{path: path, name: filepath.Base(path), nodeType: nodeType, size: size, children: map[string]*snapshotTreeNode{}} + if path == "/" { + node.name = "/" + } + nodes[path] = node + parent.children[path] = node + return node + } + for _, item := range items { + path := snapshotItemPath(item) + if path == "/" { + continue + } + nodeType := "file" + if value, _ := item["type"].(string); value == "dir" { + nodeType = "dir" + } + ensure(path, nodeType, snapshotItemSize(item)) + } + children := make(map[string][]SnapshotBrowserNode, len(nodes)) + for _, node := range nodes { + if len(node.children) == 0 { + continue + } + list := make([]SnapshotBrowserNode, 0, len(node.children)) + for _, child := range node.children { + list = append(list, SnapshotBrowserNode{Path: child.path, Name: child.name, Type: child.nodeType, Size: child.size, HasChildren: len(child.children) > 0}) + } + sort.Slice(list, func(i, j int) bool { + if list[i].Type != list[j].Type { + return list[i].Type == "dir" + } + return strings.ToLower(list[i].Name) < strings.ToLower(list[j].Name) + }) + children[node.path] = list + } + return snapshotTreeCache{generated: time.Now().UTC(), total: len(items), children: children} +} + +func snapshotItemPath(item map[string]any) string { + for _, key := range []string{"path", "name"} { + if value, ok := item[key].(string); ok && value != "" { + return normalizeSnapshotPath(value) + } + } + return "/" +} + +func snapshotItemSize(item map[string]any) int64 { + switch value := item["size"].(type) { + case float64: + return int64(value) + case int64: + return value + case int: + return int64(value) + default: + return 0 + } +} + +func normalizeSnapshotPath(path string) string { + path = filepath.Clean("/" + strings.TrimSpace(path)) + if path == "." { + return "/" + } + return path } func (s *Service) TestRepository(ctx context.Context, repoID string, initialize bool) error { diff --git a/plugin/urbm.plg b/plugin/urbm.plg index 55f9a77..076ef67 100644 --- a/plugin/urbm.plg +++ b/plugin/urbm.plg @@ -2,13 +2,17 @@ - + ]> +### 2026.07.10.r007 +- Serve snapshot browser folders as compact cached tree pages instead of sending the complete Restic file list to the browser. +- Prevent large snapshots with hundreds of thousands of entries from failing with generic 500 responses during snapshot browsing. + ### 2026.07.10.r006 - Stop logging high-frequency health, log, and activity polling requests unless they are slow or fail. - Log snapshot file-listing requests immediately when they start and show a larger daemon log tail in the Protokolle tab. diff --git a/webgui/URBM.page b/webgui/URBM.page index 5d468b6..13516d9 100644 --- a/webgui/URBM.page +++ b/webgui/URBM.page @@ -9,12 +9,12 @@ Tag="URBM Unraid Restic Backup Manager backup snapshots restore" - +
- -

URBM 2026.07.10.r006

Restic Backup Manager für Unraid

+ +

URBM 2026.07.10.r007

Restic Backup Manager für Unraid

Verbindung wird hergestellt...
@@ -33,4 +33,4 @@ $pluginRoot = '/plugins/urbm';
- + diff --git a/webgui/assets/urbm.js b/webgui/assets/urbm.js index 14d1657..058b5b6 100644 --- a/webgui/assets/urbm.js +++ b/webgui/assets/urbm.js @@ -8,7 +8,7 @@ const tooltip = document.getElementById('bu-tooltip'); const sourceRoots = ['/mnt/user','/mnt/disks','/mnt/remotes','/boot'].map(path=>({name:path,path})); const repositoryRoots = ['/mnt/user','/mnt/disks','/mnt/remotes'].map(path=>({name:path,path})); - const state = { config: null, runs: [], logs: null, logsLoading: false, logsError: '', repositoryStats: [], repositoryStatsLoading: false, view: 'dashboard', snapshots: [], selectedSnapshot: null, snapshotRepo: null, files: [], snapshotTree: null, snapshotTreeLoading: false, snapshotTreeError: '', snapshotTreeExpanded: {}, restoreIncludes: [], backupSelections: [], workloadSelections: [], workloads: [], workloadAvailable: true, workloadError: '', sourceTreeRoots: sourceRoots, sourceTreeChildren: {}, sourceTreeExpanded: {}, sourceTreeLoading: {}, sourceTreeErrors: {}, directoryTrees: {}, liveLogOpen: {} }; + const state = { config: null, runs: [], logs: null, logsLoading: false, logsError: '', repositoryStats: [], repositoryStatsLoading: false, view: 'dashboard', snapshots: [], selectedSnapshot: null, snapshotRepo: null, files: [], snapshotTree: null, snapshotTreeMeta: null, snapshotTreeChildren: {}, snapshotTreeLoading: false, snapshotTreeNodeLoading: {}, snapshotTreeErrors: {}, snapshotTreeError: '', snapshotTreeExpanded: {}, restoreIncludes: [], backupSelections: [], workloadSelections: [], workloads: [], workloadAvailable: true, workloadError: '', sourceTreeRoots: sourceRoots, sourceTreeChildren: {}, sourceTreeExpanded: {}, sourceTreeLoading: {}, sourceTreeErrors: {}, directoryTrees: {}, liveLogOpen: {} }; const fieldHelp = { name: 'Frei wählbarer Anzeigename. Beispiel: Appdata täglich oder VM Home Assistant.', @@ -669,34 +669,6 @@ return `

Snapshots

${button('Laden','load-snapshots','primary')}
${rows?`
${rows}
SnapshotZeitpunktHostPfade
`:'
Repository auswählen und Snapshots laden.
'}
${state.selectedSnapshot ? renderFileBrowser() : ''}`; } - function snapshotItemPath(item) { - return String(item.path || item.name || '').replace(/\/+/g,'/').replace(/\/$/,'') || '/'; - } - - function buildSnapshotTree(items) { - const nodes=new Map(); - const ensure=(path,type='dir',size=0)=>{ - path=path.replace(/\/+/g,'/').replace(/\/$/,'')||'/'; - if(nodes.has(path)) return nodes.get(path); - const parentPath=path==='/'?'':path.split('/').slice(0,-1).join('/')||'/'; - const name=path==='/'?'/':path.split('/').pop(); - const node={path,name,type,size:Number(size||0),children:[]}; - nodes.set(path,node); - if(path!=='/') ensure(parentPath).children.push(node); - return node; - }; - ensure('/'); - items.forEach(item=>{ - const path=snapshotItemPath(item); - if(!path||path==='/') return; - const type=item.type==='dir'?'dir':'file'; - const node=ensure(path,type,item.size); - node.type=type; node.size=Number(item.size||0); - }); - nodes.forEach(node=>node.children.sort((a,b)=>(a.type===b.type?a.name.localeCompare(b.name):a.type==='dir'?-1:1))); - return nodes.get('/'); - } - function snapshotSelectionState(path) { const exact=state.restoreIncludes.includes(path); const prefix=path.replace(/\/$/,'')+'/'; @@ -708,18 +680,31 @@ const selection=snapshotSelectionState(node.path); const isDir=node.type==='dir'; const expanded=state.snapshotTreeExpanded[node.path]===true; - const toggle=isDir?``:''; + const canExpand=isDir&&node.hasChildren; + const loading=state.snapshotTreeNodeLoading[node.path]===true; + const children=state.snapshotTreeChildren[node.path]||[]; + const error=state.snapshotTreeErrors[node.path]||''; + const toggle=canExpand?``:''; const label=selection.partial?'Teilweise ausgewählt':''; - let html=`
${toggle}
`; - if(isDir&&expanded) html+=`
${node.children.length?node.children.map(child=>snapshotTreeNode(child,depth+1)).join(''):'
Keine Einträge.
'}
`; + let html=`
${toggle}
`; + if(canExpand&&expanded) { + let body=''; + if(loading) body='
Snapshot-Ordner wird geladen…
'; + else if(error) body=`
${esc(error)} ${button('Erneut versuchen',`retry-snapshot-tree:${encodeURIComponent(node.path)}`)}
`; + else body=children.length?children.map(child=>snapshotTreeNode(child,depth+1)).join(''):'
Keine Einträge.
'; + html+=`
${body}
`; + } return html+'
'; } function renderFileBrowser() { const chosen = state.restoreIncludes.length ? state.restoreIncludes.map(path=>`${esc(path)}`).join('') : 'Nichts ausgewählt.'; - const loading='
Snapshot-Struktur wird vollständig eingelesen…Kein Prozentwert verfügbar, Restic liefert die Dateiliste erst am Ende zurück.
'; + const loading='
Snapshot-Index wird eingelesen…Der Daemon baut daraus intern einen Baum. Danach werden nur sichtbare Ordner übertragen.
'; const body=state.snapshotTreeLoading?loading:state.snapshotTreeError?`
${esc(state.snapshotTreeError)}
`:state.snapshotTree?snapshotTreeNode(state.snapshotTree,0):'
Keine Dateien im Snapshot gefunden.
'; - return `

Snapshot ${esc(state.selectedSnapshot.slice(0,12))}

${state.files.length?`${state.files.length.toLocaleString()} Einträge vorgeladen`:'Struktur noch nicht geladen'}
${button(`Auswahl wiederherstellen (${state.restoreIncludes.length})`,'restore-selected','primary')}
${chosen}
${body}
`; + const total=state.snapshotTreeMeta?.total||0; + const loaded=Object.values(state.snapshotTreeChildren).reduce((sum,items)=>sum+items.length,0); + const hint=total?`${Number(total).toLocaleString()} Einträge im Snapshot · ${Number(loaded).toLocaleString()} sichtbare Einträge geladen`:'Struktur noch nicht geladen'; + return `

Snapshot ${esc(state.selectedSnapshot.slice(0,12))}

${esc(hint)}
${button(`Auswahl wiederherstellen (${state.restoreIncludes.length})`,'restore-selected','primary')}
${chosen}
${body}
`; } function renderRestore() { @@ -792,6 +777,14 @@ render(); restoreScrollState(scrollSnapshot, content); } + async function loadSnapshotChildren(path) { + const query = path && path !== '/' ? `?path=${encodeURIComponent(path)}` : ''; + const page = await api(`/v1/repositories/${state.snapshotRepo}/snapshots/${state.selectedSnapshot}/files${query}`); + const items = Array.isArray(page) ? page : (page.items || []); + state.snapshotTreeChildren[path || '/'] = items; + state.snapshotTreeMeta = Array.isArray(page) ? { total: items.length } : page; + return items; + } const lines = (value) => value.split('\n').map(v=>v.trim()).filter(Boolean); const runTime = run => new Date(run.createdAt || run.startedAt || run.finishedAt || 0).getTime() || 0; const snapshotTime = snapshot => new Date(snapshot.time || 0).getTime() || 0; @@ -822,13 +815,14 @@ if (name === 'maint') { await api(`/v1/repositories/${b}/${a}`,{method:'POST'}); notify(`${a==='check'?'Prüfung':'Bereinigung'} wurde eingereiht`); return load(); } if (name === 'repo-snapshots') { state.view='snapshots'; document.querySelector('[data-view="snapshots"]').click(); render(); setTimeout(()=>{document.getElementById('bu-snapshot-repo').value=a; handle('load-snapshots');},0); return; } if (name === 'load-snapshots') { const repo=document.getElementById('bu-snapshot-repo').value; state.snapshots=newestSnapshots(await api(`/v1/repositories/${repo}/snapshots`)); state.snapshotRepo=repo; return render(); } - if (name === 'browse') { state.selectedSnapshot=a; state.restoreIncludes=[]; state.files=[]; state.snapshotTree=null; state.snapshotTreeError=''; state.snapshotTreeExpanded={'/':true}; state.snapshotTreeLoading=true; render(); try { state.files=await api(`/v1/repositories/${state.snapshotRepo}/snapshots/${a}/files`); state.snapshotTree=buildSnapshotTree(state.files); } catch(error) { state.snapshotTreeError=`Snapshot-Struktur konnte nicht geladen werden: ${error.message}. Details stehen im Tab Protokolle.`; } finally { state.snapshotTreeLoading=false; return render(); } } + if (name === 'browse') { state.selectedSnapshot=a; state.restoreIncludes=[]; state.files=[]; state.snapshotTree={path:'/',name:'/',type:'dir',hasChildren:true}; state.snapshotTreeMeta=null; state.snapshotTreeChildren={}; state.snapshotTreeNodeLoading={}; state.snapshotTreeErrors={}; state.snapshotTreeError=''; state.snapshotTreeExpanded={'/':true}; state.snapshotTreeLoading=true; render(); try { state.files=await loadSnapshotChildren('/'); } catch(error) { state.snapshotTreeError=`Snapshot-Struktur konnte nicht geladen werden: ${error.message}. Details stehen im Tab Protokolle.`; } finally { state.snapshotTreeLoading=false; return render(); } } if (name === 'toggle-source-tree') { return toggleSourceTree(decodeURIComponent(a)); } if (name === 'retry-source-tree') { const path=decodeURIComponent(a); delete state.sourceTreeChildren[path]; state.sourceTreeErrors[path]=''; return toggleSourceTree(path,true); } if (name === 'toggle-dir-tree') { return toggleDirectoryTree(a, decodeURIComponent(b)); } if (name === 'retry-dir-tree') { const path=decodeURIComponent(b); const tree=state.directoryTrees[a]; if(tree) { delete tree.children[path]; tree.errors[path]=''; } return toggleDirectoryTree(a,path,true); } if (name === 'select-dir-tree') { selectDirectoryTreePath(a, decodeURIComponent(b)); return; } - if (name === 'toggle-snapshot-tree') { const path=decodeURIComponent(a); state.snapshotTreeExpanded[path]=!state.snapshotTreeExpanded[path]; return renderPreservingScroll(); } + if (name === 'toggle-snapshot-tree') { const path=decodeURIComponent(a); if(state.snapshotTreeExpanded[path]) { state.snapshotTreeExpanded[path]=false; return renderPreservingScroll(); } state.snapshotTreeExpanded[path]=true; renderPreservingScroll(); if(Object.prototype.hasOwnProperty.call(state.snapshotTreeChildren,path)&&!state.snapshotTreeErrors[path]) return; state.snapshotTreeNodeLoading[path]=true; state.snapshotTreeErrors[path]=''; renderPreservingScroll(); try { await loadSnapshotChildren(path); } catch(error) { state.snapshotTreeErrors[path]=`Snapshot-Ordner konnte nicht geladen werden: ${error.message}`; } finally { state.snapshotTreeNodeLoading[path]=false; return renderPreservingScroll(); } } + if (name === 'retry-snapshot-tree') { const path=decodeURIComponent(a); delete state.snapshotTreeChildren[path]; state.snapshotTreeErrors[path]=''; state.snapshotTreeExpanded[path]=false; return handle(`toggle-snapshot-tree:${encodeURIComponent(path)}`); } if (name === 'restore-selected') { if(!state.restoreIncludes.length) throw new Error('Wähle mindestens eine Datei oder einen Ordner aus.'); state.view='restore'; document.querySelectorAll('.bu-tabs button').forEach(x=>x.classList.toggle('active',x.dataset.view==='restore')); return render(); } if (name === 'delete-job') { if(confirm('Diesen Job löschen?')) { state.config.jobs=state.config.jobs.filter(j=>j.id!==a); await saveConfig(); render(); } return; } if (name === 'duplicate-job') { const source=state.config.jobs.find(j=>j.id===a); const copy=structuredClone(source); copy.id=id('job'); copy.name += ' Kopie'; copy.enabled=false; state.config.jobs.push(copy); await saveConfig(); return render(); }