diff --git a/dist/urbm-2026.07.13.r010-x86_64-1.txz.sha256 b/dist/urbm-2026.07.13.r010-x86_64-1.txz.sha256 deleted file mode 100644 index e90fe19..0000000 --- a/dist/urbm-2026.07.13.r010-x86_64-1.txz.sha256 +++ /dev/null @@ -1 +0,0 @@ -3f655152c8e0875711c8b02036b05b1a163227a658fe6b2e4bf91397270a7f48 urbm-2026.07.13.r010-x86_64-1.txz diff --git a/dist/urbm-2026.07.13.r010-x86_64-1.txz b/dist/urbm-2026.07.13.r011-x86_64-1.txz similarity index 56% rename from dist/urbm-2026.07.13.r010-x86_64-1.txz rename to dist/urbm-2026.07.13.r011-x86_64-1.txz index 3ac4416..1eaeecd 100644 Binary files a/dist/urbm-2026.07.13.r010-x86_64-1.txz and b/dist/urbm-2026.07.13.r011-x86_64-1.txz differ diff --git a/dist/urbm-2026.07.13.r011-x86_64-1.txz.sha256 b/dist/urbm-2026.07.13.r011-x86_64-1.txz.sha256 new file mode 100644 index 0000000..85f1813 --- /dev/null +++ b/dist/urbm-2026.07.13.r011-x86_64-1.txz.sha256 @@ -0,0 +1 @@ +eca5aa766ba47c6582d25b64ccbf9ae351aa712bba04bbe2d9b0fc44fa53fb25 urbm-2026.07.13.r011-x86_64-1.txz diff --git a/dist/urbm.plg b/dist/urbm.plg index 9c0389e..e474e66 100644 --- a/dist/urbm.plg +++ b/dist/urbm.plg @@ -2,13 +2,18 @@ - + - + ]> +### 2026.07.13.r011 +- Use durable backup measurements when deciding whether a scheduled job needs a catch-up run, even after activity history was cleared. +- Append one compact JSON-lines record per completed backup instead of rewriting the full metric history for every queue state change. +- Migrate the existing `backup-metrics.json` automatically to `backup-metrics.jsonl` and compact in batches after the file exceeds 5,500 records while retaining the newest 5,000. + ### 2026.07.13.r010 - Return an empty JSON array instead of `null` when no dashboard backup measurements exist yet. - Make the WebGUI tolerate empty or invalid metric responses so the rest of the interface remains available. diff --git a/internal/service/safety_test.go b/internal/service/safety_test.go index 6135754..bb9e45d 100644 --- a/internal/service/safety_test.go +++ b/internal/service/safety_test.go @@ -9,6 +9,7 @@ import ( "time" "git.casaderoll.de/michael/urbm/internal/model" + "git.casaderoll.de/michael/urbm/internal/queue" "git.casaderoll.de/michael/urbm/internal/store" ) @@ -39,7 +40,8 @@ func TestRepositoryLockHonorsContextCancellation(t *testing.T) { } func TestBackupMetricsSurviveClearedRunSnapshot(t *testing.T) { - st := store.New(t.TempDir()) + dir := t.TempDir() + st := store.New(dir) if err := st.Init(); err != nil { t.Fatal(err) } @@ -47,13 +49,36 @@ func TestBackupMetricsSurviveClearedRunSnapshot(t *testing.T) { finished := time.Date(2026, 7, 13, 1, 30, 0, 0, time.UTC) run := model.Run{ID: "run-1", JobID: "job-1", TaskType: "backup", Status: "success", FinishedAt: &finished, BytesProcessed: 1024, FilesProcessed: 12} s.queuePersistence([]model.Run{run}) + before, err := os.ReadFile(filepath.Join(dir, "backup-metrics.jsonl")) + if err != nil { + t.Fatal(err) + } s.queuePersistence(nil) + after, err := os.ReadFile(filepath.Join(dir, "backup-metrics.jsonl")) + if err != nil { + t.Fatal(err) + } + if string(after) != string(before) { + t.Fatal("clearing activities rewrote the backup metric file") + } metrics := s.BackupMetrics() if len(metrics) != 1 || metrics[0].RunID != run.ID || metrics[0].FilesProcessed != 12 { t.Fatalf("backup metrics after clear = %#v", metrics) } } +func TestLastRunUsesDurableMetricAfterActivityClear(t *testing.T) { + created := time.Date(2026, 7, 13, 1, 0, 0, 0, time.UTC) + finished := created.Add(3 * time.Hour) + q := queue.New(nil, nil) + q.RestoreHistory([]model.Run{{ID: "run-1", JobID: "job-1", Status: "success", CreatedAt: created}}) + s := &Service{queue: q, backupMetrics: []model.BackupMetric{{RunID: "run-1", JobID: "job-1", Status: "success", FinishedAt: finished}}} + q.ClearHistory() + if got := s.LastRun("job-1"); !got.Equal(finished) { + t.Fatalf("last run after activity clear = %v, want %v", got, finished) + } +} + func TestBackupMetricsOnlyCaptureCompletedSuccessfulBackupsOnce(t *testing.T) { st := store.New(t.TempDir()) s := &Service{store: st} diff --git a/internal/service/service.go b/internal/service/service.go index f256752..0cc9d19 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -54,8 +54,15 @@ type Service struct { persistStopOnce sync.Once metricsMu sync.RWMutex backupMetrics []model.BackupMetric + metricRecords int + metricsAppendOK bool } +const ( + maxBackupMetrics = 5000 + compactBackupMetricsAt = 5500 +) + type SnapshotBrowserNode struct { Path string `json:"path"` Name string `json:"name"` @@ -101,10 +108,20 @@ func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runn store: st, secrets: secrets, restic: rr, rsync: rs, mounts: mounts, workloads: workloads, notifier: notifier, log: log, config: config, repositoryLocks: map[string]chan struct{}{}, persistRuns: make(chan []model.Run, 1), persistStop: make(chan struct{}), persistDone: make(chan struct{}), } - if metrics, err := st.LoadBackupMetrics(); err != nil { + if metrics, legacy, err := st.LoadBackupMetrics(); err != nil { log.Error("load backup metrics", "error", err) } else { - s.backupMetrics = metrics + s.backupMetrics = normalizeBackupMetrics(metrics) + s.metricRecords = len(s.backupMetrics) + if legacy || len(metrics) != len(s.backupMetrics) { + if err := st.SaveBackupMetrics(s.backupMetrics); err != nil { + log.Error("migrate backup metrics to append format", "error", err) + } else { + s.metricsAppendOK = true + } + } else { + s.metricsAppendOK = true + } } go s.runPersistence() s.queue = queue.New(s.execute, s.queuePersistence) @@ -202,9 +219,6 @@ func (s *Service) saveRunState(runs []model.Run) { if err := s.persistRunLogs(runs); err != nil { s.log.Error("save persistent run logs", "error", err) } - if err := s.store.SaveBackupMetrics(s.BackupMetrics()); err != nil { - s.log.Error("save backup metrics", "error", err) - } } func (s *Service) captureBackupMetrics(runs []model.Run) { @@ -212,11 +226,11 @@ func (s *Service) captureBackupMetrics(runs []model.Run) { return } s.metricsMu.Lock() - defer s.metricsMu.Unlock() known := make(map[string]struct{}, len(s.backupMetrics)) for _, metric := range s.backupMetrics { known[metric.RunID] = struct{}{} } + added := make([]model.BackupMetric, 0, 1) for _, run := range runs { if run.TaskType != "backup" || run.FinishedAt == nil || (run.Status != "success" && run.Status != "warning") { continue @@ -224,17 +238,58 @@ func (s *Service) captureBackupMetrics(runs []model.Run) { if _, exists := known[run.ID]; exists { continue } - s.backupMetrics = append(s.backupMetrics, model.BackupMetric{ + metric := model.BackupMetric{ SchemaVersion: model.SchemaVersion, RunID: run.ID, JobID: run.JobID, SnapshotID: run.SnapshotID, FinishedAt: *run.FinishedAt, Status: run.Status, BytesProcessed: run.BytesProcessed, FilesProcessed: run.FilesProcessed, BytesAdded: run.BytesAdded, FilesNew: run.FilesNew, FilesChanged: run.FilesChanged, - }) + } + s.backupMetrics = append(s.backupMetrics, metric) + added = append(added, metric) known[run.ID] = struct{}{} } - sort.SliceStable(s.backupMetrics, func(i, j int) bool { return s.backupMetrics[i].FinishedAt.Before(s.backupMetrics[j].FinishedAt) }) - if len(s.backupMetrics) > 5000 { - s.backupMetrics = append([]model.BackupMetric(nil), s.backupMetrics[len(s.backupMetrics)-5000:]...) + if len(added) == 0 { + s.metricsMu.Unlock() + return } + s.backupMetrics = normalizeBackupMetrics(s.backupMetrics) + if len(added) == 1 && s.metricsAppendOK && s.metricRecords < compactBackupMetricsAt { + if err := s.store.AppendBackupMetric(added[0]); err == nil { + s.metricRecords++ + s.metricsMu.Unlock() + return + } else if s.log != nil { + s.log.Error("append backup metric", "error", err) + } + } + if err := s.store.SaveBackupMetrics(s.backupMetrics); err != nil { + if s.log != nil { + s.log.Error("compact backup metrics", "error", err) + } + } else { + s.metricRecords = len(s.backupMetrics) + s.metricsAppendOK = true + } + s.metricsMu.Unlock() +} + +func normalizeBackupMetrics(metrics []model.BackupMetric) []model.BackupMetric { + known := make(map[string]struct{}, len(metrics)) + normalized := make([]model.BackupMetric, 0, len(metrics)) + for _, metric := range metrics { + if metric.RunID == "" { + continue + } + if _, exists := known[metric.RunID]; exists { + continue + } + known[metric.RunID] = struct{}{} + normalized = append(normalized, metric) + } + sort.SliceStable(normalized, func(i, j int) bool { return normalized[i].FinishedAt.Before(normalized[j].FinishedAt) }) + if len(normalized) > maxBackupMetrics { + normalized = append([]model.BackupMetric(nil), normalized[len(normalized)-maxBackupMetrics:]...) + } + return normalized } func (s *Service) persistRunLogs(runs []model.Run) error { @@ -353,6 +408,13 @@ func (s *Service) LastRun(jobID string) time.Time { latest = run.CreatedAt } } + s.metricsMu.RLock() + defer s.metricsMu.RUnlock() + for _, metric := range s.backupMetrics { + if metric.JobID == jobID && metric.FinishedAt.After(latest) { + latest = metric.FinishedAt + } + } return latest } diff --git a/internal/store/store.go b/internal/store/store.go index f153d19..2b46689 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "encoding/json" "errors" "fmt" @@ -70,16 +71,48 @@ func (s *Store) SaveRuns(runs []model.Run) error { return writeJSONAtomic(s.runsPath(), runs, 0600) } -func (s *Store) LoadBackupMetrics() ([]model.BackupMetric, error) { +func (s *Store) LoadBackupMetrics() ([]model.BackupMetric, bool, error) { s.mu.RLock() defer s.mu.RUnlock() - var metrics []model.BackupMetric - if err := readJSON(s.backupMetricsPath(), &metrics); errors.Is(err, os.ErrNotExist) { - return []model.BackupMetric{}, nil - } else if err != nil { - return nil, err + path := s.backupMetricsPath() + b, err := os.ReadFile(path) + legacySource := false + if errors.Is(err, os.ErrNotExist) { + path = s.legacyBackupMetricsPath() + b, err = os.ReadFile(path) + legacySource = true + if errors.Is(err, os.ErrNotExist) { + return []model.BackupMetric{}, false, nil + } } - return metrics, nil + if err != nil { + return nil, false, err + } + b = bytes.TrimSpace(b) + if len(b) == 0 { + return []model.BackupMetric{}, false, nil + } + if b[0] == '[' { + var metrics []model.BackupMetric + if err := json.Unmarshal(b, &metrics); err != nil { + return nil, true, fmt.Errorf("decode %s: %w", path, err) + } + return metrics, true, nil + } + lines := bytes.Split(b, []byte{'\n'}) + metrics := make([]model.BackupMetric, 0, len(lines)) + for index, line := range lines { + line = bytes.TrimSpace(line) + if len(line) == 0 { + continue + } + var metric model.BackupMetric + if err := json.Unmarshal(line, &metric); err != nil { + return nil, legacySource, fmt.Errorf("decode %s line %d: %w", path, index+1, err) + } + metrics = append(metrics, metric) + } + return metrics, legacySource, nil } func (s *Store) SaveBackupMetrics(metrics []model.BackupMetric) error { @@ -88,12 +121,45 @@ func (s *Store) SaveBackupMetrics(metrics []model.BackupMetric) error { if len(metrics) > 5000 { metrics = metrics[len(metrics)-5000:] } - return writeJSONAtomic(s.backupMetricsPath(), metrics, 0600) + var data bytes.Buffer + encoder := json.NewEncoder(&data) + for _, metric := range metrics { + if err := encoder.Encode(metric); err != nil { + return err + } + } + return writeBytesAtomic(s.backupMetricsPath(), data.Bytes(), 0600) +} + +func (s *Store) AppendBackupMetric(metric model.BackupMetric) error { + s.mu.Lock() + defer s.mu.Unlock() + b, err := json.Marshal(metric) + if err != nil { + return err + } + b = append(b, '\n') + file, err := os.OpenFile(s.backupMetricsPath(), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + return err + } + if _, err := file.Write(b); err != nil { + file.Close() + return err + } + if err := file.Sync(); err != nil { + file.Close() + return err + } + return file.Close() } func (s *Store) configPath() string { return filepath.Join(s.dir, "config.json") } func (s *Store) runsPath() string { return filepath.Join(s.dir, "runs.json") } func (s *Store) backupMetricsPath() string { + return filepath.Join(s.dir, "backup-metrics.jsonl") +} +func (s *Store) legacyBackupMetricsPath() string { return filepath.Join(s.dir, "backup-metrics.json") } @@ -114,6 +180,10 @@ func writeJSONAtomic(path string, value any, mode os.FileMode) error { return err } b = append(b, '\n') + return writeBytesAtomic(path, b, mode) +} + +func writeBytesAtomic(path string, b []byte, mode os.FileMode) error { tmp, err := os.CreateTemp(filepath.Dir(path), ".urbm-*") if err != nil { return err diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 9810bd0..ddd08ef 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -43,10 +43,13 @@ func TestBackupMetricsPersistSeparatelyFromRuns(t *testing.T) { if err := s.SaveRuns(nil); err != nil { t.Fatal(err) } - loaded, err := s.LoadBackupMetrics() + loaded, legacy, err := s.LoadBackupMetrics() if err != nil { t.Fatal(err) } + if legacy { + t.Fatal("compacted metric file was detected as legacy JSON array") + } if len(loaded) != 1 || loaded[0].RunID != "run-1" || loaded[0].BytesProcessed != 42 { t.Fatalf("backup metrics = %#v", loaded) } @@ -58,3 +61,32 @@ func TestBackupMetricsPersistSeparatelyFromRuns(t *testing.T) { t.Fatalf("backup metrics mode = %o", info.Mode().Perm()) } } + +func TestBackupMetricsMigrateFromJSONArrayAndAppend(t *testing.T) { + s := New(t.TempDir()) + finished := time.Date(2026, 7, 13, 1, 30, 0, 0, time.UTC) + legacyJSON := `[{"schemaVersion":1,"runId":"old","jobId":"job-1","finishedAt":"2026-07-13T01:30:00Z","status":"success","bytesProcessed":42,"filesProcessed":7}]` + if err := os.WriteFile(s.legacyBackupMetricsPath(), []byte(legacyJSON), 0600); err != nil { + t.Fatal(err) + } + metrics, legacy, err := s.LoadBackupMetrics() + if err != nil { + t.Fatal(err) + } + if !legacy || len(metrics) != 1 || metrics[0].RunID != "old" { + t.Fatalf("legacy metrics = %#v, legacy = %v", metrics, legacy) + } + if err := s.SaveBackupMetrics(metrics); err != nil { + t.Fatal(err) + } + if err := s.AppendBackupMetric(model.BackupMetric{SchemaVersion: model.SchemaVersion, RunID: "new", JobID: "job-1", FinishedAt: finished.Add(time.Hour), Status: "success"}); err != nil { + t.Fatal(err) + } + metrics, legacy, err = s.LoadBackupMetrics() + if err != nil { + t.Fatal(err) + } + if legacy || len(metrics) != 2 || metrics[1].RunID != "new" { + t.Fatalf("appended metrics = %#v, legacy = %v", metrics, legacy) + } +} diff --git a/plugin/doinst.sh b/plugin/doinst.sh index 73614cd..8f512f3 100755 --- a/plugin/doinst.sh +++ b/plugin/doinst.sh @@ -12,7 +12,7 @@ fi mkdir -p "$CONFIG" "$ROOT_PREFIX/run/urbm" "$ROOT_PREFIX/var/lib/urbm" if [ -d "$OLD_CONFIG" ]; then - for item in config.json runs.json backup-metrics.json secrets; do + for item in config.json runs.json backup-metrics.json backup-metrics.jsonl secrets; do if [ -e "$OLD_CONFIG/$item" ] && [ ! -e "$CONFIG/$item" ]; then cp -a "$OLD_CONFIG/$item" "$CONFIG/$item" fi diff --git a/plugin/urbm.plg b/plugin/urbm.plg index cfb5e42..bb6ee97 100644 --- a/plugin/urbm.plg +++ b/plugin/urbm.plg @@ -2,13 +2,18 @@ - + ]> +### 2026.07.13.r011 +- Use durable backup measurements when deciding whether a scheduled job needs a catch-up run, even after activity history was cleared. +- Append one compact JSON-lines record per completed backup instead of rewriting the full metric history for every queue state change. +- Migrate the existing `backup-metrics.json` automatically to `backup-metrics.jsonl` and compact in batches after the file exceeds 5,500 records while retaining the newest 5,000. + ### 2026.07.13.r010 - Return an empty JSON array instead of `null` when no dashboard backup measurements exist yet. - Make the WebGUI tolerate empty or invalid metric responses so the rest of the interface remains available. diff --git a/tests/migration_test.sh b/tests/migration_test.sh index f518d0d..afe54a6 100755 --- a/tests/migration_test.sh +++ b/tests/migration_test.sh @@ -11,6 +11,7 @@ mkdir -p "$OLD/secrets" "$ROOT/var/log/plugins" "$ROOT/tmp/plugins" printf '%s\n' '{"settings":{"resticPath":"/usr/local/libexec/'"$OLD_NAME"'/restic","restoreRoot":"/mnt/user/'"$OLD_NAME"'-restores","persistentLogDir":"/mnt/user/system/'"$OLD_NAME"'-logs"}}' > "$OLD/config.json" printf '%s\n' '[]' > "$OLD/runs.json" printf '%s\n' '[]' > "$OLD/backup-metrics.json" +printf '%s\n' '{"runId":"new-format"}' > "$OLD/backup-metrics.jsonl" printf '%s\n' 'encrypted' > "$OLD/secrets/secret.json" touch "$ROOT/boot/config/plugins/$OLD_NAME.plg" "$ROOT/var/log/plugins/$OLD_NAME.plg" "$ROOT/tmp/plugins/$OLD_NAME.plg" @@ -19,6 +20,7 @@ URBM_ROOT_PREFIX="$ROOT" "$(dirname "$0")/../plugin/doinst.sh" test -f "$NEW/config.json" test -f "$NEW/runs.json" test -f "$NEW/backup-metrics.json" +test -f "$NEW/backup-metrics.jsonl" test -f "$NEW/secrets/secret.json" test ! -e "$OLD" test ! -e "$ROOT/boot/config/plugins/$OLD_NAME.plg" diff --git a/webgui/URBM.page b/webgui/URBM.page index 84cf0b1..c140e6f 100644 --- a/webgui/URBM.page +++ b/webgui/URBM.page @@ -8,7 +8,7 @@ Tag="URBM Unraid Restic Backup Manager backup snapshots restore" ---