michael/URBM
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stream snapshot file listings

Browse Source
This commit is contained in:
Mikei386
2026-07-10 12:41:48 +02:00
parent c2f0f0b277
commit f22c2e462d
10 changed files with 59 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dist/urbm-2026.07.10.r002-x86_64-1.txz.sha256
Vendored
-1
View File
@@ -1 +0,0 @@
ec72032752aaae7a2bde7587781f65b2b5b4d9ef74a6aad79a3e121491b30958 urbm-2026.07.10.r002-x86_64-1.txz
dist/urbm-2026.07.10.r002-x86_64-1.txz → dist/urbm-2026.07.10.r003-x86_64-1.txz
Vendored
BIN
View File
Binary file not shown.
dist/urbm-2026.07.10.r003-x86_64-1.txz.sha256
Vendored
+1
View File
@@ -0,0 +1 @@
e08f4f2ee2a9589531051a519eceed485dde2d0b456e4e9bf160dd9796be2a8e urbm-2026.07.10.r003-x86_64-1.txz
dist/urbm.plg
Vendored
+7 -2
View File
@@ -2,13 +2,18 @@
<!DOCTYPE PLUGIN [
<!ENTITY name "urbm">
<!ENTITY author "Michael Roll">
<!ENTITY version "2026.07.10.r002">
<!ENTITY version "2026.07.10.r003">
<!ENTITY pluginURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm.plg">
<!ENTITY packageURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm-&version;-x86_64-1.txz">
<!ENTITY packageSHA256 "ec72032752aaae7a2bde7587781f65b2b5b4d9ef74a6aad79a3e121491b30958">
<!ENTITY packageSHA256 "e08f4f2ee2a9589531051a519eceed485dde2d0b456e4e9bf160dd9796be2a8e">
]>
<PLUGIN name="&name;" author="&author;" version="&version;" pluginURL="&pluginURL;" min="7.0.0" support="https://git.casaderoll.de/michael/URBM/issues" icon="urbm.png">
<CHANGES>
### 2026.07.10.r003
- Stream Restic snapshot file listings instead of buffering the full `restic ls --json` output in memory.
- Extend snapshot file-listing timeouts across the daemon and PHP bridge to reduce generic 500 errors on large snapshots.
- Return a clearer timeout error when Restic snapshot listing exceeds the daemon limit.
### 2026.07.10.r002
- Include retention results in backup notifications after Restic forget runs, including removed and remaining snapshot counts when available.
- Include repository size before, after, and freed space in prune notifications when Restic statistics are available.
internal/api/server.go
+1 -1
View File
@@ -268,7 +268,7 @@ func (s *Server) repositoryStats(w http.ResponseWriter, r *http.Request) {
writeJSON(w, 200, s.service.RepositoryStats(ctx))
}
func (s *Server) snapshotFiles(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute)
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Minute)
defer cancel()
items, err := s.service.SnapshotFiles(ctx, r.PathValue("id"), r.PathValue("snapshot"), r.URL.Query().Get("path"))
if err != nil {
internal/restic/restic.go
+8 -9
View File
@@ -274,20 +274,16 @@ func (r *Runner) List(ctx context.Context, repo model.Repository, snapshot, path
if path != "" {
args = append(args, path)
}
var output []byte
if err := r.run(ctx, repo, args, nil, &output); err != nil {
return nil, err
}
items := []map[string]any{}
scanner := bufio.NewScanner(strings.NewReader(string(output)))
scanner.Buffer(make([]byte, 0, 64*1024), 2*1024*1024)
for scanner.Scan() {
if err := r.run(ctx, repo, args, func(line []byte) {
var item map[string]any
if json.Unmarshal(scanner.Bytes(), &item) == nil && item["struct_type"] == "node" {
if json.Unmarshal(line, &item) == nil && item["struct_type"] == "node" {
items = append(items, item)
}
}, nil); err != nil {
return nil, err
}
return items, scanner.Err()
return items, nil
}
func (r *Runner) Diff(ctx context.Context, repo model.Repository, before, after string, limit int) (DiffResult, error) {
@@ -466,6 +462,9 @@ func (r *Runner) runWithInputEnvPassword(ctx context.Context, repo model.Reposit
if errors.Is(ctx.Err(), context.Canceled) {
return fmt.Errorf("cancelled: %w", ctx.Err())
}
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
return fmt.Errorf("environment: Restic-Aufruf hat das Zeitlimit überschritten: %w", ctx.Err())
}
return resticError(message, err)
}
return nil
internal/restic/restic_test.go
+30
View File
@@ -233,6 +233,36 @@ printf '%%s\n' 'Files: 1 new, 1 removed, 2 changed'
}
}
func TestListStreamsSnapshotNodes(t *testing.T) {
dir := t.TempDir()
argsPath := filepath.Join(dir, "args")
script := filepath.Join(dir, "restic")
body := fmt.Sprintf(`#!/bin/sh
printf '%%s\n' "$@" > '%s'
printf '%%s\n' '{"struct_type":"snapshot","id":"snap"}'
printf '%%s\n' '{"struct_type":"node","path":"/mnt/user","type":"dir"}'
printf '%%s\n' '{"struct_type":"node","path":"/mnt/user/file.txt","type":"file","size":12}'
`, argsPath)
if err := os.WriteFile(script, []byte(body), 0700); err != nil {
t.Fatal(err)
}
runner := &Runner{Binary: script, RuntimeDir: dir, Secrets: fakeSecrets{"password": "secret"}}
repo := model.Repository{Type: model.RepositoryLocal, Location: "/repo", PasswordRef: "password"}
items, err := runner.List(context.Background(), repo, "snap", "/mnt/user")
if err != nil {
t.Fatal(err)
}
if len(items) != 2 || items[0]["path"] != "/mnt/user" || items[1]["path"] != "/mnt/user/file.txt" {
t.Fatalf("unexpected items: %+v", items)
}
args, _ := os.ReadFile(argsPath)
for _, expected := range []string{"ls", "snap", "--json", "/mnt/user"} {
if !strings.Contains(string(args), expected) {
t.Fatalf("arguments missing %q: %s", expected, args)
}
}
}
func TestStatsUsesRequestedRepositoryCountingMode(t *testing.T) {
dir := t.TempDir()
argsPath := filepath.Join(dir, "args")
plugin/urbm.plg
+6 -1
View File
@@ -2,13 +2,18 @@
<!DOCTYPE PLUGIN [
<!ENTITY name "urbm">
<!ENTITY author "Michael Roll">
<!ENTITY version "2026.07.10.r002">
<!ENTITY version "2026.07.10.r003">
<!ENTITY pluginURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm.plg">
<!ENTITY packageURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm-&version;-x86_64-1.txz">
<!ENTITY packageSHA256 "REPLACE_DURING_RELEASE">
]>
<PLUGIN name="&name;" author="&author;" version="&version;" pluginURL="&pluginURL;" min="7.0.0" support="https://git.casaderoll.de/michael/URBM/issues" icon="urbm.png">
<CHANGES>
### 2026.07.10.r003
- Stream Restic snapshot file listings instead of buffering the full `restic ls --json` output in memory.
- Extend snapshot file-listing timeouts across the daemon and PHP bridge to reduce generic 500 errors on large snapshots.
- Return a clearer timeout error when Restic snapshot listing exceeds the daemon limit.
### 2026.07.10.r002
- Include retention results in backup notifications after Restic forget runs, including removed and remaining snapshot counts when available.
- Include repository size before, after, and freed space in prune notifications when Restic statistics are available.
webgui/URBM.page
+4 -4
View File
@@ -9,12 +9,12 @@ Tag="URBM Unraid Restic Backup Manager backup snapshots restore"
<?php
$pluginRoot = '/plugins/urbm';
?>
<link rel="stylesheet" href="<?= $pluginRoot ?>/assets/urbm.css?v=20260710r002">
<link rel="stylesheet" href="<?= $pluginRoot ?>/assets/urbm.css?v=20260710r003">
<div id="urbm-app" data-api="<?= $pluginRoot ?>/api.php">
<header class="bu-header">
<div class="bu-brand">
<img class="bu-logo" src="<?= $pluginRoot ?>/images/urbm.png?v=20260710r002" alt="URBM-Logo">
<div><h1>URBM <span class="bu-version">2026.07.10.r002</span></h1><p>Restic Backup Manager für Unraid</p></div>
<img class="bu-logo" src="<?= $pluginRoot ?>/images/urbm.png?v=20260710r003" alt="URBM-Logo">
<div><h1>URBM <span class="bu-version">2026.07.10.r003</span></h1><p>Restic Backup Manager für Unraid</p></div>
</div>
<div id="bu-health" class="bu-health" tabindex="0" data-tooltip="Zeigt, ob die URBM-Hintergrundkomponente erreichbar ist. Beispiel: 'Daemon online' bedeutet, dass Jobs gestartet werden können.">Verbindung wird hergestellt...</div>
</header>
@@ -32,4 +32,4 @@ $pluginRoot = '/plugins/urbm';
<div id="bu-tooltip" role="tooltip" aria-hidden="true"></div>
</div>
<script>window.URBM_CSRF = <?= json_encode($var['csrf_token'] ?? '') ?>;</script>
<script src="<?= $pluginRoot ?>/assets/urbm-2026.07.10.r002.js"></script>
<script src="<?= $pluginRoot ?>/assets/urbm-2026.07.10.r003.js"></script>
webgui/api.php
+2 -1
View File
@@ -6,6 +6,7 @@ header('Content-Type: application/json');
$socket = '/run/urbm/urbm.sock';
$method = $_SERVER['REQUEST_METHOD'] ?? 'GET';
$path = $_GET['path'] ?? '/v1/health';
$isSnapshotFileRequest = preg_match('#/snapshots/[^/]+/files(?:\?|$)#', $path) === 1;
if (!preg_match('#^/v1/[A-Za-z0-9_./?=&%:-]*$#', $path) || str_contains($path, '..')) {
http_response_code(400);
@@ -26,7 +27,7 @@ curl_setopt_array($curl, [
CURLOPT_CUSTOMREQUEST => $method,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_CONNECTTIMEOUT => 3,
CURLOPT_TIMEOUT => 310,
CURLOPT_TIMEOUT => $isSnapshotFileRequest ? 1250 : 310,
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
]);