Stream snapshot file listings
This commit is contained in:
@@ -1 +0,0 @@
|
|||||||
ec72032752aaae7a2bde7587781f65b2b5b4d9ef74a6aad79a3e121491b30958 urbm-2026.07.10.r002-x86_64-1.txz
|
|
||||||
BIN
Binary file not shown.
@@ -0,0 +1 @@
|
|||||||
|
e08f4f2ee2a9589531051a519eceed485dde2d0b456e4e9bf160dd9796be2a8e urbm-2026.07.10.r003-x86_64-1.txz
|
||||||
Vendored
+7
-2
@@ -2,13 +2,18 @@
|
|||||||
<!DOCTYPE PLUGIN [
|
<!DOCTYPE PLUGIN [
|
||||||
<!ENTITY name "urbm">
|
<!ENTITY name "urbm">
|
||||||
<!ENTITY author "Michael Roll">
|
<!ENTITY author "Michael Roll">
|
||||||
<!ENTITY version "2026.07.10.r002">
|
<!ENTITY version "2026.07.10.r003">
|
||||||
<!ENTITY pluginURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm.plg">
|
<!ENTITY pluginURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm.plg">
|
||||||
<!ENTITY packageURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm-&version;-x86_64-1.txz">
|
<!ENTITY packageURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm-&version;-x86_64-1.txz">
|
||||||
<!ENTITY packageSHA256 "ec72032752aaae7a2bde7587781f65b2b5b4d9ef74a6aad79a3e121491b30958">
|
<!ENTITY packageSHA256 "e08f4f2ee2a9589531051a519eceed485dde2d0b456e4e9bf160dd9796be2a8e">
|
||||||
]>
|
]>
|
||||||
<PLUGIN name="&name;" author="&author;" version="&version;" pluginURL="&pluginURL;" min="7.0.0" support="https://git.casaderoll.de/michael/URBM/issues" icon="urbm.png">
|
<PLUGIN name="&name;" author="&author;" version="&version;" pluginURL="&pluginURL;" min="7.0.0" support="https://git.casaderoll.de/michael/URBM/issues" icon="urbm.png">
|
||||||
<CHANGES>
|
<CHANGES>
|
||||||
|
### 2026.07.10.r003
|
||||||
|
- Stream Restic snapshot file listings instead of buffering the full `restic ls --json` output in memory.
|
||||||
|
- Extend snapshot file-listing timeouts across the daemon and PHP bridge to reduce generic 500 errors on large snapshots.
|
||||||
|
- Return a clearer timeout error when Restic snapshot listing exceeds the daemon limit.
|
||||||
|
|
||||||
### 2026.07.10.r002
|
### 2026.07.10.r002
|
||||||
- Include retention results in backup notifications after Restic forget runs, including removed and remaining snapshot counts when available.
|
- Include retention results in backup notifications after Restic forget runs, including removed and remaining snapshot counts when available.
|
||||||
- Include repository size before, after, and freed space in prune notifications when Restic statistics are available.
|
- Include repository size before, after, and freed space in prune notifications when Restic statistics are available.
|
||||||
|
|||||||
@@ -268,7 +268,7 @@ func (s *Server) repositoryStats(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeJSON(w, 200, s.service.RepositoryStats(ctx))
|
writeJSON(w, 200, s.service.RepositoryStats(ctx))
|
||||||
}
|
}
|
||||||
func (s *Server) snapshotFiles(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) snapshotFiles(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute)
|
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Minute)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
items, err := s.service.SnapshotFiles(ctx, r.PathValue("id"), r.PathValue("snapshot"), r.URL.Query().Get("path"))
|
items, err := s.service.SnapshotFiles(ctx, r.PathValue("id"), r.PathValue("snapshot"), r.URL.Query().Get("path"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -274,20 +274,16 @@ func (r *Runner) List(ctx context.Context, repo model.Repository, snapshot, path
|
|||||||
if path != "" {
|
if path != "" {
|
||||||
args = append(args, path)
|
args = append(args, path)
|
||||||
}
|
}
|
||||||
var output []byte
|
|
||||||
if err := r.run(ctx, repo, args, nil, &output); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
items := []map[string]any{}
|
items := []map[string]any{}
|
||||||
scanner := bufio.NewScanner(strings.NewReader(string(output)))
|
if err := r.run(ctx, repo, args, func(line []byte) {
|
||||||
scanner.Buffer(make([]byte, 0, 64*1024), 2*1024*1024)
|
|
||||||
for scanner.Scan() {
|
|
||||||
var item map[string]any
|
var item map[string]any
|
||||||
if json.Unmarshal(scanner.Bytes(), &item) == nil && item["struct_type"] == "node" {
|
if json.Unmarshal(line, &item) == nil && item["struct_type"] == "node" {
|
||||||
items = append(items, item)
|
items = append(items, item)
|
||||||
}
|
}
|
||||||
|
}, nil); err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
return items, scanner.Err()
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) Diff(ctx context.Context, repo model.Repository, before, after string, limit int) (DiffResult, error) {
|
func (r *Runner) Diff(ctx context.Context, repo model.Repository, before, after string, limit int) (DiffResult, error) {
|
||||||
@@ -466,6 +462,9 @@ func (r *Runner) runWithInputEnvPassword(ctx context.Context, repo model.Reposit
|
|||||||
if errors.Is(ctx.Err(), context.Canceled) {
|
if errors.Is(ctx.Err(), context.Canceled) {
|
||||||
return fmt.Errorf("cancelled: %w", ctx.Err())
|
return fmt.Errorf("cancelled: %w", ctx.Err())
|
||||||
}
|
}
|
||||||
|
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||||
|
return fmt.Errorf("environment: Restic-Aufruf hat das Zeitlimit überschritten: %w", ctx.Err())
|
||||||
|
}
|
||||||
return resticError(message, err)
|
return resticError(message, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -233,6 +233,36 @@ printf '%%s\n' 'Files: 1 new, 1 removed, 2 changed'
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestListStreamsSnapshotNodes(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
argsPath := filepath.Join(dir, "args")
|
||||||
|
script := filepath.Join(dir, "restic")
|
||||||
|
body := fmt.Sprintf(`#!/bin/sh
|
||||||
|
printf '%%s\n' "$@" > '%s'
|
||||||
|
printf '%%s\n' '{"struct_type":"snapshot","id":"snap"}'
|
||||||
|
printf '%%s\n' '{"struct_type":"node","path":"/mnt/user","type":"dir"}'
|
||||||
|
printf '%%s\n' '{"struct_type":"node","path":"/mnt/user/file.txt","type":"file","size":12}'
|
||||||
|
`, argsPath)
|
||||||
|
if err := os.WriteFile(script, []byte(body), 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
runner := &Runner{Binary: script, RuntimeDir: dir, Secrets: fakeSecrets{"password": "secret"}}
|
||||||
|
repo := model.Repository{Type: model.RepositoryLocal, Location: "/repo", PasswordRef: "password"}
|
||||||
|
items, err := runner.List(context.Background(), repo, "snap", "/mnt/user")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(items) != 2 || items[0]["path"] != "/mnt/user" || items[1]["path"] != "/mnt/user/file.txt" {
|
||||||
|
t.Fatalf("unexpected items: %+v", items)
|
||||||
|
}
|
||||||
|
args, _ := os.ReadFile(argsPath)
|
||||||
|
for _, expected := range []string{"ls", "snap", "--json", "/mnt/user"} {
|
||||||
|
if !strings.Contains(string(args), expected) {
|
||||||
|
t.Fatalf("arguments missing %q: %s", expected, args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestStatsUsesRequestedRepositoryCountingMode(t *testing.T) {
|
func TestStatsUsesRequestedRepositoryCountingMode(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
argsPath := filepath.Join(dir, "args")
|
argsPath := filepath.Join(dir, "args")
|
||||||
|
|||||||
+6
-1
@@ -2,13 +2,18 @@
|
|||||||
<!DOCTYPE PLUGIN [
|
<!DOCTYPE PLUGIN [
|
||||||
<!ENTITY name "urbm">
|
<!ENTITY name "urbm">
|
||||||
<!ENTITY author "Michael Roll">
|
<!ENTITY author "Michael Roll">
|
||||||
<!ENTITY version "2026.07.10.r002">
|
<!ENTITY version "2026.07.10.r003">
|
||||||
<!ENTITY pluginURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm.plg">
|
<!ENTITY pluginURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm.plg">
|
||||||
<!ENTITY packageURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm-&version;-x86_64-1.txz">
|
<!ENTITY packageURL "https://git.casaderoll.de/michael/URBM/raw/branch/main/dist/urbm-&version;-x86_64-1.txz">
|
||||||
<!ENTITY packageSHA256 "REPLACE_DURING_RELEASE">
|
<!ENTITY packageSHA256 "REPLACE_DURING_RELEASE">
|
||||||
]>
|
]>
|
||||||
<PLUGIN name="&name;" author="&author;" version="&version;" pluginURL="&pluginURL;" min="7.0.0" support="https://git.casaderoll.de/michael/URBM/issues" icon="urbm.png">
|
<PLUGIN name="&name;" author="&author;" version="&version;" pluginURL="&pluginURL;" min="7.0.0" support="https://git.casaderoll.de/michael/URBM/issues" icon="urbm.png">
|
||||||
<CHANGES>
|
<CHANGES>
|
||||||
|
### 2026.07.10.r003
|
||||||
|
- Stream Restic snapshot file listings instead of buffering the full `restic ls --json` output in memory.
|
||||||
|
- Extend snapshot file-listing timeouts across the daemon and PHP bridge to reduce generic 500 errors on large snapshots.
|
||||||
|
- Return a clearer timeout error when Restic snapshot listing exceeds the daemon limit.
|
||||||
|
|
||||||
### 2026.07.10.r002
|
### 2026.07.10.r002
|
||||||
- Include retention results in backup notifications after Restic forget runs, including removed and remaining snapshot counts when available.
|
- Include retention results in backup notifications after Restic forget runs, including removed and remaining snapshot counts when available.
|
||||||
- Include repository size before, after, and freed space in prune notifications when Restic statistics are available.
|
- Include repository size before, after, and freed space in prune notifications when Restic statistics are available.
|
||||||
|
|||||||
+4
-4
@@ -9,12 +9,12 @@ Tag="URBM Unraid Restic Backup Manager backup snapshots restore"
|
|||||||
<?php
|
<?php
|
||||||
$pluginRoot = '/plugins/urbm';
|
$pluginRoot = '/plugins/urbm';
|
||||||
?>
|
?>
|
||||||
<link rel="stylesheet" href="<?= $pluginRoot ?>/assets/urbm.css?v=20260710r002">
|
<link rel="stylesheet" href="<?= $pluginRoot ?>/assets/urbm.css?v=20260710r003">
|
||||||
<div id="urbm-app" data-api="<?= $pluginRoot ?>/api.php">
|
<div id="urbm-app" data-api="<?= $pluginRoot ?>/api.php">
|
||||||
<header class="bu-header">
|
<header class="bu-header">
|
||||||
<div class="bu-brand">
|
<div class="bu-brand">
|
||||||
<img class="bu-logo" src="<?= $pluginRoot ?>/images/urbm.png?v=20260710r002" alt="URBM-Logo">
|
<img class="bu-logo" src="<?= $pluginRoot ?>/images/urbm.png?v=20260710r003" alt="URBM-Logo">
|
||||||
<div><h1>URBM <span class="bu-version">2026.07.10.r002</span></h1><p>Restic Backup Manager für Unraid</p></div>
|
<div><h1>URBM <span class="bu-version">2026.07.10.r003</span></h1><p>Restic Backup Manager für Unraid</p></div>
|
||||||
</div>
|
</div>
|
||||||
<div id="bu-health" class="bu-health" tabindex="0" data-tooltip="Zeigt, ob die URBM-Hintergrundkomponente erreichbar ist. Beispiel: 'Daemon online' bedeutet, dass Jobs gestartet werden können.">Verbindung wird hergestellt...</div>
|
<div id="bu-health" class="bu-health" tabindex="0" data-tooltip="Zeigt, ob die URBM-Hintergrundkomponente erreichbar ist. Beispiel: 'Daemon online' bedeutet, dass Jobs gestartet werden können.">Verbindung wird hergestellt...</div>
|
||||||
</header>
|
</header>
|
||||||
@@ -32,4 +32,4 @@ $pluginRoot = '/plugins/urbm';
|
|||||||
<div id="bu-tooltip" role="tooltip" aria-hidden="true"></div>
|
<div id="bu-tooltip" role="tooltip" aria-hidden="true"></div>
|
||||||
</div>
|
</div>
|
||||||
<script>window.URBM_CSRF = <?= json_encode($var['csrf_token'] ?? '') ?>;</script>
|
<script>window.URBM_CSRF = <?= json_encode($var['csrf_token'] ?? '') ?>;</script>
|
||||||
<script src="<?= $pluginRoot ?>/assets/urbm-2026.07.10.r002.js"></script>
|
<script src="<?= $pluginRoot ?>/assets/urbm-2026.07.10.r003.js"></script>
|
||||||
|
|||||||
+2
-1
@@ -6,6 +6,7 @@ header('Content-Type: application/json');
|
|||||||
$socket = '/run/urbm/urbm.sock';
|
$socket = '/run/urbm/urbm.sock';
|
||||||
$method = $_SERVER['REQUEST_METHOD'] ?? 'GET';
|
$method = $_SERVER['REQUEST_METHOD'] ?? 'GET';
|
||||||
$path = $_GET['path'] ?? '/v1/health';
|
$path = $_GET['path'] ?? '/v1/health';
|
||||||
|
$isSnapshotFileRequest = preg_match('#/snapshots/[^/]+/files(?:\?|$)#', $path) === 1;
|
||||||
|
|
||||||
if (!preg_match('#^/v1/[A-Za-z0-9_./?=&%:-]*$#', $path) || str_contains($path, '..')) {
|
if (!preg_match('#^/v1/[A-Za-z0-9_./?=&%:-]*$#', $path) || str_contains($path, '..')) {
|
||||||
http_response_code(400);
|
http_response_code(400);
|
||||||
@@ -26,7 +27,7 @@ curl_setopt_array($curl, [
|
|||||||
CURLOPT_CUSTOMREQUEST => $method,
|
CURLOPT_CUSTOMREQUEST => $method,
|
||||||
CURLOPT_RETURNTRANSFER => true,
|
CURLOPT_RETURNTRANSFER => true,
|
||||||
CURLOPT_CONNECTTIMEOUT => 3,
|
CURLOPT_CONNECTTIMEOUT => 3,
|
||||||
CURLOPT_TIMEOUT => 310,
|
CURLOPT_TIMEOUT => $isSnapshotFileRequest ? 1250 : 310,
|
||||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
|
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user