Persist dashboard backup metrics separately
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.casaderoll.de/michael/urbm/internal/model"
|
||||
"git.casaderoll.de/michael/urbm/internal/store"
|
||||
)
|
||||
|
||||
func TestRepositoryLockHonorsContextCancellation(t *testing.T) {
|
||||
@@ -25,6 +26,40 @@ func TestRepositoryLockHonorsContextCancellation(t *testing.T) {
|
||||
unlock()
|
||||
}
|
||||
|
||||
func TestBackupMetricsSurviveClearedRunSnapshot(t *testing.T) {
|
||||
st := store.New(t.TempDir())
|
||||
if err := st.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := &Service{store: st, persistRuns: make(chan []model.Run, 1)}
|
||||
finished := time.Date(2026, 7, 13, 1, 30, 0, 0, time.UTC)
|
||||
run := model.Run{ID: "run-1", JobID: "job-1", TaskType: "backup", Status: "success", FinishedAt: &finished, BytesProcessed: 1024, FilesProcessed: 12}
|
||||
s.queuePersistence([]model.Run{run})
|
||||
s.queuePersistence(nil)
|
||||
metrics := s.BackupMetrics()
|
||||
if len(metrics) != 1 || metrics[0].RunID != run.ID || metrics[0].FilesProcessed != 12 {
|
||||
t.Fatalf("backup metrics after clear = %#v", metrics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupMetricsOnlyCaptureCompletedSuccessfulBackupsOnce(t *testing.T) {
|
||||
st := store.New(t.TempDir())
|
||||
s := &Service{store: st}
|
||||
finished := time.Now().UTC()
|
||||
valid := model.Run{ID: "valid", JobID: "job-1", TaskType: "backup", Status: "warning", FinishedAt: &finished}
|
||||
s.captureBackupMetrics([]model.Run{
|
||||
valid,
|
||||
valid,
|
||||
{ID: "failed", TaskType: "backup", Status: "failed", FinishedAt: &finished},
|
||||
{ID: "active", TaskType: "backup", Status: "running"},
|
||||
{ID: "rsync", TaskType: "rsync", Status: "success", FinishedAt: &finished},
|
||||
})
|
||||
metrics := s.BackupMetrics()
|
||||
if len(metrics) != 1 || metrics[0].RunID != valid.ID {
|
||||
t.Fatalf("captured backup metrics = %#v", metrics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistentRunLogsAreWrittenAndPruned(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
config := model.DefaultConfig()
|
||||
|
||||
@@ -52,6 +52,8 @@ type Service struct {
|
||||
persistStop chan struct{}
|
||||
persistDone chan struct{}
|
||||
persistStopOnce sync.Once
|
||||
metricsMu sync.RWMutex
|
||||
backupMetrics []model.BackupMetric
|
||||
}
|
||||
|
||||
type SnapshotBrowserNode struct {
|
||||
@@ -99,6 +101,11 @@ func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runn
|
||||
store: st, secrets: secrets, restic: rr, rsync: rs, mounts: mounts, workloads: workloads, notifier: notifier, log: log, config: config,
|
||||
repositoryLocks: map[string]chan struct{}{}, persistRuns: make(chan []model.Run, 1), persistStop: make(chan struct{}), persistDone: make(chan struct{}),
|
||||
}
|
||||
if metrics, err := st.LoadBackupMetrics(); err != nil {
|
||||
log.Error("load backup metrics", "error", err)
|
||||
} else {
|
||||
s.backupMetrics = metrics
|
||||
}
|
||||
go s.runPersistence()
|
||||
s.queue = queue.New(s.execute, s.queuePersistence)
|
||||
if runs, err := st.LoadRuns(); err == nil {
|
||||
@@ -124,6 +131,11 @@ func (s *Service) Wait(ctx context.Context) error {
|
||||
}
|
||||
func (s *Service) Config() model.Config { s.mu.RLock(); defer s.mu.RUnlock(); return s.config }
|
||||
func (s *Service) Runs() []model.Run { return s.queue.Snapshot() }
|
||||
func (s *Service) BackupMetrics() []model.BackupMetric {
|
||||
s.metricsMu.RLock()
|
||||
defer s.metricsMu.RUnlock()
|
||||
return append([]model.BackupMetric(nil), s.backupMetrics...)
|
||||
}
|
||||
func (s *Service) Logs() map[string]any {
|
||||
return map[string]any{
|
||||
"daemon": tailFile("/var/log/urbm.log", 1024*1024, 1000),
|
||||
@@ -143,6 +155,7 @@ func runsWithLogs(runs []model.Run, limit int) []model.Run {
|
||||
}
|
||||
|
||||
func (s *Service) queuePersistence(runs []model.Run) {
|
||||
s.captureBackupMetrics(runs)
|
||||
select {
|
||||
case s.persistRuns <- runs:
|
||||
return
|
||||
@@ -187,6 +200,39 @@ func (s *Service) saveRunState(runs []model.Run) {
|
||||
if err := s.persistRunLogs(runs); err != nil {
|
||||
s.log.Error("save persistent run logs", "error", err)
|
||||
}
|
||||
if err := s.store.SaveBackupMetrics(s.BackupMetrics()); err != nil {
|
||||
s.log.Error("save backup metrics", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) captureBackupMetrics(runs []model.Run) {
|
||||
if s.store == nil {
|
||||
return
|
||||
}
|
||||
s.metricsMu.Lock()
|
||||
defer s.metricsMu.Unlock()
|
||||
known := make(map[string]struct{}, len(s.backupMetrics))
|
||||
for _, metric := range s.backupMetrics {
|
||||
known[metric.RunID] = struct{}{}
|
||||
}
|
||||
for _, run := range runs {
|
||||
if run.TaskType != "backup" || run.FinishedAt == nil || (run.Status != "success" && run.Status != "warning") {
|
||||
continue
|
||||
}
|
||||
if _, exists := known[run.ID]; exists {
|
||||
continue
|
||||
}
|
||||
s.backupMetrics = append(s.backupMetrics, model.BackupMetric{
|
||||
SchemaVersion: model.SchemaVersion, RunID: run.ID, JobID: run.JobID, SnapshotID: run.SnapshotID,
|
||||
FinishedAt: *run.FinishedAt, Status: run.Status, BytesProcessed: run.BytesProcessed,
|
||||
FilesProcessed: run.FilesProcessed, BytesAdded: run.BytesAdded, FilesNew: run.FilesNew, FilesChanged: run.FilesChanged,
|
||||
})
|
||||
known[run.ID] = struct{}{}
|
||||
}
|
||||
sort.SliceStable(s.backupMetrics, func(i, j int) bool { return s.backupMetrics[i].FinishedAt.Before(s.backupMetrics[j].FinishedAt) })
|
||||
if len(s.backupMetrics) > 5000 {
|
||||
s.backupMetrics = append([]model.BackupMetric(nil), s.backupMetrics[len(s.backupMetrics)-5000:]...)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) persistRunLogs(runs []model.Run) error {
|
||||
|
||||
Reference in New Issue
Block a user