Reduce metric writes and preserve scheduler history

internal/service/service.go

@@ -54,8 +54,15 @@ type Service struct {
 	persistStopOnce sync.Once
 	metricsMu       sync.RWMutex
 	backupMetrics   []model.BackupMetric
+	metricRecords   int
+	metricsAppendOK bool
 }
 
+const (
+	maxBackupMetrics       = 5000
+	compactBackupMetricsAt = 5500
+)
+
 type SnapshotBrowserNode struct {
 	Path        string `json:"path"`
 	Name        string `json:"name"`
@@ -101,10 +108,20 @@ func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runn
 		store: st, secrets: secrets, restic: rr, rsync: rs, mounts: mounts, workloads: workloads, notifier: notifier, log: log, config: config,
 		repositoryLocks: map[string]chan struct{}{}, persistRuns: make(chan []model.Run, 1), persistStop: make(chan struct{}), persistDone: make(chan struct{}),
 	}
-	if metrics, err := st.LoadBackupMetrics(); err != nil {
+	if metrics, legacy, err := st.LoadBackupMetrics(); err != nil {
 		log.Error("load backup metrics", "error", err)
 	} else {
-		s.backupMetrics = metrics
+		s.backupMetrics = normalizeBackupMetrics(metrics)
+		s.metricRecords = len(s.backupMetrics)
+		if legacy || len(metrics) != len(s.backupMetrics) {
+			if err := st.SaveBackupMetrics(s.backupMetrics); err != nil {
+				log.Error("migrate backup metrics to append format", "error", err)
+			} else {
+				s.metricsAppendOK = true
+			}
+		} else {
+			s.metricsAppendOK = true
+		}
 	}
 	go s.runPersistence()
 	s.queue = queue.New(s.execute, s.queuePersistence)
@@ -202,9 +219,6 @@ func (s *Service) saveRunState(runs []model.Run) {
 	if err := s.persistRunLogs(runs); err != nil {
 		s.log.Error("save persistent run logs", "error", err)
 	}
-	if err := s.store.SaveBackupMetrics(s.BackupMetrics()); err != nil {
-		s.log.Error("save backup metrics", "error", err)
-	}
 }
 
 func (s *Service) captureBackupMetrics(runs []model.Run) {
@@ -212,11 +226,11 @@ func (s *Service) captureBackupMetrics(runs []model.Run) {
 		return
 	}
 	s.metricsMu.Lock()
-	defer s.metricsMu.Unlock()
 	known := make(map[string]struct{}, len(s.backupMetrics))
 	for _, metric := range s.backupMetrics {
 		known[metric.RunID] = struct{}{}
 	}
+	added := make([]model.BackupMetric, 0, 1)
 	for _, run := range runs {
 		if run.TaskType != "backup" || run.FinishedAt == nil || (run.Status != "success" && run.Status != "warning") {
 			continue
@@ -224,17 +238,58 @@ func (s *Service) captureBackupMetrics(runs []model.Run) {
 		if _, exists := known[run.ID]; exists {
 			continue
 		}
-		s.backupMetrics = append(s.backupMetrics, model.BackupMetric{
+		metric := model.BackupMetric{
 			SchemaVersion: model.SchemaVersion, RunID: run.ID, JobID: run.JobID, SnapshotID: run.SnapshotID,
 			FinishedAt: *run.FinishedAt, Status: run.Status, BytesProcessed: run.BytesProcessed,
 			FilesProcessed: run.FilesProcessed, BytesAdded: run.BytesAdded, FilesNew: run.FilesNew, FilesChanged: run.FilesChanged,
-		})
+		}
+		s.backupMetrics = append(s.backupMetrics, metric)
+		added = append(added, metric)
 		known[run.ID] = struct{}{}
 	}
-	sort.SliceStable(s.backupMetrics, func(i, j int) bool { return s.backupMetrics[i].FinishedAt.Before(s.backupMetrics[j].FinishedAt) })
-	if len(s.backupMetrics) > 5000 {
-		s.backupMetrics = append([]model.BackupMetric(nil), s.backupMetrics[len(s.backupMetrics)-5000:]...)
+	if len(added) == 0 {
+		s.metricsMu.Unlock()
+		return
 	}
+	s.backupMetrics = normalizeBackupMetrics(s.backupMetrics)
+	if len(added) == 1 && s.metricsAppendOK && s.metricRecords < compactBackupMetricsAt {
+		if err := s.store.AppendBackupMetric(added[0]); err == nil {
+			s.metricRecords++
+			s.metricsMu.Unlock()
+			return
+		} else if s.log != nil {
+			s.log.Error("append backup metric", "error", err)
+		}
+	}
+	if err := s.store.SaveBackupMetrics(s.backupMetrics); err != nil {
+		if s.log != nil {
+			s.log.Error("compact backup metrics", "error", err)
+		}
+	} else {
+		s.metricRecords = len(s.backupMetrics)
+		s.metricsAppendOK = true
+	}
+	s.metricsMu.Unlock()
+}
+
+func normalizeBackupMetrics(metrics []model.BackupMetric) []model.BackupMetric {
+	known := make(map[string]struct{}, len(metrics))
+	normalized := make([]model.BackupMetric, 0, len(metrics))
+	for _, metric := range metrics {
+		if metric.RunID == "" {
+			continue
+		}
+		if _, exists := known[metric.RunID]; exists {
+			continue
+		}
+		known[metric.RunID] = struct{}{}
+		normalized = append(normalized, metric)
+	}
+	sort.SliceStable(normalized, func(i, j int) bool { return normalized[i].FinishedAt.Before(normalized[j].FinishedAt) })
+	if len(normalized) > maxBackupMetrics {
+		normalized = append([]model.BackupMetric(nil), normalized[len(normalized)-maxBackupMetrics:]...)
+	}
+	return normalized
 }
 
 func (s *Service) persistRunLogs(runs []model.Run) error {
@@ -353,6 +408,13 @@ func (s *Service) LastRun(jobID string) time.Time {
 			latest = run.CreatedAt
 		}
 	}
+	s.metricsMu.RLock()
+	defer s.metricsMu.RUnlock()
+	for _, metric := range s.backupMetrics {
+		if metric.JobID == jobID && metric.FinishedAt.After(latest) {
+			latest = metric.FinishedAt
+		}
+	}
 	return latest
 }