Reduce metric writes and preserve scheduler history

2026-07-13 20:53:16 +02:00
parent ea97d7dbc9
commit 819055cbf0
12 changed files with 228 additions and 27 deletions
@@ -9,6 +9,7 @@ import (
 	"time"

 	"git.casaderoll.de/michael/urbm/internal/model"
+	"git.casaderoll.de/michael/urbm/internal/queue"
 	"git.casaderoll.de/michael/urbm/internal/store"
 )

@@ -39,7 +40,8 @@ func TestRepositoryLockHonorsContextCancellation(t *testing.T) {
 }

 func TestBackupMetricsSurviveClearedRunSnapshot(t *testing.T) {
-	st := store.New(t.TempDir())
+	dir := t.TempDir()
+	st := store.New(dir)
 	if err := st.Init(); err != nil {
 		t.Fatal(err)
 	}
@@ -47,13 +49,36 @@ func TestBackupMetricsSurviveClearedRunSnapshot(t *testing.T) {
 	finished := time.Date(2026, 7, 13, 1, 30, 0, 0, time.UTC)
 	run := model.Run{ID: "run-1", JobID: "job-1", TaskType: "backup", Status: "success", FinishedAt: &finished, BytesProcessed: 1024, FilesProcessed: 12}
 	s.queuePersistence([]model.Run{run})
+	before, err := os.ReadFile(filepath.Join(dir, "backup-metrics.jsonl"))
+	if err != nil {
+		t.Fatal(err)
+	}
 	s.queuePersistence(nil)
+	after, err := os.ReadFile(filepath.Join(dir, "backup-metrics.jsonl"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(after) != string(before) {
+		t.Fatal("clearing activities rewrote the backup metric file")
+	}
 	metrics := s.BackupMetrics()
 	if len(metrics) != 1 || metrics[0].RunID != run.ID || metrics[0].FilesProcessed != 12 {
 		t.Fatalf("backup metrics after clear = %#v", metrics)
 	}
 }

+func TestLastRunUsesDurableMetricAfterActivityClear(t *testing.T) {
+	created := time.Date(2026, 7, 13, 1, 0, 0, 0, time.UTC)
+	finished := created.Add(3 * time.Hour)
+	q := queue.New(nil, nil)
+	q.RestoreHistory([]model.Run{{ID: "run-1", JobID: "job-1", Status: "success", CreatedAt: created}})
+	s := &Service{queue: q, backupMetrics: []model.BackupMetric{{RunID: "run-1", JobID: "job-1", Status: "success", FinishedAt: finished}}}
+	q.ClearHistory()
+	if got := s.LastRun("job-1"); !got.Equal(finished) {
+		t.Fatalf("last run after activity clear = %v, want %v", got, finished)
+	}
+}
+
 func TestBackupMetricsOnlyCaptureCompletedSuccessfulBackupsOnce(t *testing.T) {
 	st := store.New(t.TempDir())
 	s := &Service{store: st}
@@ -54,8 +54,15 @@ type Service struct {
 	persistStopOnce sync.Once
 	metricsMu       sync.RWMutex
 	backupMetrics   []model.BackupMetric
+	metricRecords   int
+	metricsAppendOK bool
 }

+const (
+	maxBackupMetrics       = 5000
+	compactBackupMetricsAt = 5500
+)
+
 type SnapshotBrowserNode struct {
 	Path        string `json:"path"`
 	Name        string `json:"name"`
@@ -101,10 +108,20 @@ func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runn
 		store: st, secrets: secrets, restic: rr, rsync: rs, mounts: mounts, workloads: workloads, notifier: notifier, log: log, config: config,
 		repositoryLocks: map[string]chan struct{}{}, persistRuns: make(chan []model.Run, 1), persistStop: make(chan struct{}), persistDone: make(chan struct{}),
 	}
-	if metrics, err := st.LoadBackupMetrics(); err != nil {
+	if metrics, legacy, err := st.LoadBackupMetrics(); err != nil {
 		log.Error("load backup metrics", "error", err)
 	} else {
-		s.backupMetrics = metrics
+		s.backupMetrics = normalizeBackupMetrics(metrics)
+		s.metricRecords = len(s.backupMetrics)
+		if legacy || len(metrics) != len(s.backupMetrics) {
+			if err := st.SaveBackupMetrics(s.backupMetrics); err != nil {
+				log.Error("migrate backup metrics to append format", "error", err)
+			} else {
+				s.metricsAppendOK = true
+			}
+		} else {
+			s.metricsAppendOK = true
+		}
 	}
 	go s.runPersistence()
 	s.queue = queue.New(s.execute, s.queuePersistence)
@@ -202,9 +219,6 @@ func (s *Service) saveRunState(runs []model.Run) {
 	if err := s.persistRunLogs(runs); err != nil {
 		s.log.Error("save persistent run logs", "error", err)
 	}
-	if err := s.store.SaveBackupMetrics(s.BackupMetrics()); err != nil {
-		s.log.Error("save backup metrics", "error", err)
-	}
 }

 func (s *Service) captureBackupMetrics(runs []model.Run) {
@@ -212,11 +226,11 @@ func (s *Service) captureBackupMetrics(runs []model.Run) {
 		return
 	}
 	s.metricsMu.Lock()
-	defer s.metricsMu.Unlock()
 	known := make(map[string]struct{}, len(s.backupMetrics))
 	for _, metric := range s.backupMetrics {
 		known[metric.RunID] = struct{}{}
 	}
+	added := make([]model.BackupMetric, 0, 1)
 	for _, run := range runs {
 		if run.TaskType != "backup" || run.FinishedAt == nil || (run.Status != "success" && run.Status != "warning") {
 			continue
@@ -224,17 +238,58 @@ func (s *Service) captureBackupMetrics(runs []model.Run) {
 		if _, exists := known[run.ID]; exists {
 			continue
 		}
-		s.backupMetrics = append(s.backupMetrics, model.BackupMetric{
+		metric := model.BackupMetric{
 			SchemaVersion: model.SchemaVersion, RunID: run.ID, JobID: run.JobID, SnapshotID: run.SnapshotID,
 			FinishedAt: *run.FinishedAt, Status: run.Status, BytesProcessed: run.BytesProcessed,
 			FilesProcessed: run.FilesProcessed, BytesAdded: run.BytesAdded, FilesNew: run.FilesNew, FilesChanged: run.FilesChanged,
-		})
+		}
+		s.backupMetrics = append(s.backupMetrics, metric)
+		added = append(added, metric)
 		known[run.ID] = struct{}{}
 	}
-	sort.SliceStable(s.backupMetrics, func(i, j int) bool { return s.backupMetrics[i].FinishedAt.Before(s.backupMetrics[j].FinishedAt) })
-	if len(s.backupMetrics) > 5000 {
-		s.backupMetrics = append([]model.BackupMetric(nil), s.backupMetrics[len(s.backupMetrics)-5000:]...)
+	if len(added) == 0 {
+		s.metricsMu.Unlock()
+		return
 	}
+	s.backupMetrics = normalizeBackupMetrics(s.backupMetrics)
+	if len(added) == 1 && s.metricsAppendOK && s.metricRecords < compactBackupMetricsAt {
+		if err := s.store.AppendBackupMetric(added[0]); err == nil {
+			s.metricRecords++
+			s.metricsMu.Unlock()
+			return
+		} else if s.log != nil {
+			s.log.Error("append backup metric", "error", err)
+		}
+	}
+	if err := s.store.SaveBackupMetrics(s.backupMetrics); err != nil {
+		if s.log != nil {
+			s.log.Error("compact backup metrics", "error", err)
+		}
+	} else {
+		s.metricRecords = len(s.backupMetrics)
+		s.metricsAppendOK = true
+	}
+	s.metricsMu.Unlock()
+}
+
+func normalizeBackupMetrics(metrics []model.BackupMetric) []model.BackupMetric {
+	known := make(map[string]struct{}, len(metrics))
+	normalized := make([]model.BackupMetric, 0, len(metrics))
+	for _, metric := range metrics {
+		if metric.RunID == "" {
+			continue
+		}
+		if _, exists := known[metric.RunID]; exists {
+			continue
+		}
+		known[metric.RunID] = struct{}{}
+		normalized = append(normalized, metric)
+	}
+	sort.SliceStable(normalized, func(i, j int) bool { return normalized[i].FinishedAt.Before(normalized[j].FinishedAt) })
+	if len(normalized) > maxBackupMetrics {
+		normalized = append([]model.BackupMetric(nil), normalized[len(normalized)-maxBackupMetrics:]...)
+	}
+	return normalized
 }

 func (s *Service) persistRunLogs(runs []model.Run) error {
@@ -353,6 +408,13 @@ func (s *Service) LastRun(jobID string) time.Time {
 			latest = run.CreatedAt
 		}
 	}
+	s.metricsMu.RLock()
+	defer s.metricsMu.RUnlock()
+	for _, metric := range s.backupMetrics {
+		if metric.JobID == jobID && metric.FinishedAt.After(latest) {
+			latest = metric.FinishedAt
+		}
+	}
 	return latest
 }