Improve dashboard health and state recovery

This commit is contained in:
Mikei386
2026-07-13 21:06:09 +02:00
parent 819055cbf0
commit 8f75a04db6
15 changed files with 313 additions and 38 deletions
+24
View File
@@ -79,6 +79,30 @@ func TestLastRunUsesDurableMetricAfterActivityClear(t *testing.T) {
}
}
func TestDashboardStatusUsesOnlyBackupsAndFindsOverdueJobs(t *testing.T) {
now := time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC)
lastSuccess := now.Add(-30 * time.Hour)
failedAt := now.Add(-2 * time.Hour)
q := queue.New(nil, nil)
q.RestoreHistory([]model.Run{
{ID: "failed-backup", JobID: "job-1", TaskType: "backup", Status: "failed", CreatedAt: failedAt},
{ID: "successful-check", JobID: "repo-1", TaskType: "check", Status: "success", CreatedAt: now.Add(-time.Hour)},
})
config := model.DefaultConfig()
config.Jobs = []model.Job{{ID: "job-1", Name: "Daily", Type: model.JobShare, Enabled: true, Schedule: model.Schedule{Cron: "0 2 * * *", Timezone: "UTC"}}}
s := &Service{queue: q, config: config, backupMetrics: []model.BackupMetric{{RunID: "success", JobID: "job-1", Status: "success", FinishedAt: lastSuccess}}}
status := s.DashboardStatus(now)
if status.LastSuccessfulBackup == nil || status.LastSuccessfulBackup.RunID != "success" {
t.Fatalf("last successful backup = %#v", status.LastSuccessfulBackup)
}
if status.LastFailedBackup == nil || status.LastFailedBackup.ID != "failed-backup" {
t.Fatalf("last failed backup = %#v", status.LastFailedBackup)
}
if len(status.OverdueJobs) != 1 || status.OverdueJobs[0].JobID != "job-1" {
t.Fatalf("overdue jobs = %#v", status.OverdueJobs)
}
}
func TestBackupMetricsOnlyCaptureCompletedSuccessfulBackupsOnce(t *testing.T) {
st := store.New(t.TempDir())
s := &Service{store: st}
+65
View File
@@ -79,6 +79,19 @@ type SnapshotBrowserPage struct {
Generated time.Time `json:"generated"`
}
type DashboardStatus struct {
LastSuccessfulBackup *model.BackupMetric `json:"lastSuccessfulBackup,omitempty"`
LastFailedBackup *model.Run `json:"lastFailedBackup,omitempty"`
OverdueJobs []OverdueBackupJob `json:"overdueJobs"`
}
type OverdueBackupJob struct {
JobID string `json:"jobId"`
Name string `json:"name"`
ScheduledAt time.Time `json:"scheduledAt"`
LastBackupAt *time.Time `json:"lastBackupAt,omitempty"`
}
type snapshotTreeCache struct {
generated time.Time
total int
@@ -127,6 +140,8 @@ func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runn
s.queue = queue.New(s.execute, s.queuePersistence)
if runs, err := st.LoadRuns(); err == nil {
s.queue.RestoreHistory(runs)
} else {
log.Error("load run history", "error", err)
}
s.queuePersistence(s.queue.Snapshot())
return s, nil
@@ -418,6 +433,56 @@ func (s *Service) LastRun(jobID string) time.Time {
return latest
}
func (s *Service) DashboardStatus(now time.Time) DashboardStatus {
status := DashboardStatus{OverdueJobs: []OverdueBackupJob{}}
metrics := s.BackupMetrics()
lastByJob := make(map[string]time.Time)
for index := range metrics {
metric := metrics[index]
if metric.Status == "success" && (status.LastSuccessfulBackup == nil || metric.FinishedAt.After(status.LastSuccessfulBackup.FinishedAt)) {
copy := metric
status.LastSuccessfulBackup = &copy
}
if metric.FinishedAt.After(lastByJob[metric.JobID]) {
lastByJob[metric.JobID] = metric.FinishedAt
}
}
runs := s.queue.Snapshot()
activeJobs := make(map[string]bool)
for index := range runs {
run := runs[index]
if run.TaskType != "backup" {
continue
}
if run.Status == "queued" || run.Status == "running" || run.Status == "paused" {
activeJobs[run.JobID] = true
}
if run.Status == "failed" && (status.LastFailedBackup == nil || run.CreatedAt.After(status.LastFailedBackup.CreatedAt)) {
copy := run
status.LastFailedBackup = &copy
}
}
for _, job := range s.Config().Jobs {
if job.Type == model.JobRsync || !job.Enabled || job.Schedule.Cron == "" || activeJobs[job.ID] {
continue
}
scheduledAt, err := scheduler.PreviousScheduled(job.Schedule.Cron, job.Schedule.Timezone, now, 370*24*time.Hour)
if err != nil || scheduledAt.IsZero() || !lastByJob[job.ID].Before(scheduledAt) {
continue
}
overdue := OverdueBackupJob{JobID: job.ID, Name: job.Name, ScheduledAt: scheduledAt}
if last := lastByJob[job.ID]; !last.IsZero() {
lastCopy := last
overdue.LastBackupAt = &lastCopy
}
status.OverdueJobs = append(status.OverdueJobs, overdue)
}
sort.SliceStable(status.OverdueJobs, func(i, j int) bool {
return status.OverdueJobs[i].ScheduledAt.Before(status.OverdueJobs[j].ScheduledAt)
})
return status
}
func (s *Service) SaveConfig(config model.Config) error {
config = model.NormalizeConfig(config)
for _, job := range config.Jobs {