Improve dashboard health and state recovery
This commit is contained in:
@@ -79,6 +79,30 @@ func TestLastRunUsesDurableMetricAfterActivityClear(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardStatusUsesOnlyBackupsAndFindsOverdueJobs(t *testing.T) {
|
||||
now := time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC)
|
||||
lastSuccess := now.Add(-30 * time.Hour)
|
||||
failedAt := now.Add(-2 * time.Hour)
|
||||
q := queue.New(nil, nil)
|
||||
q.RestoreHistory([]model.Run{
|
||||
{ID: "failed-backup", JobID: "job-1", TaskType: "backup", Status: "failed", CreatedAt: failedAt},
|
||||
{ID: "successful-check", JobID: "repo-1", TaskType: "check", Status: "success", CreatedAt: now.Add(-time.Hour)},
|
||||
})
|
||||
config := model.DefaultConfig()
|
||||
config.Jobs = []model.Job{{ID: "job-1", Name: "Daily", Type: model.JobShare, Enabled: true, Schedule: model.Schedule{Cron: "0 2 * * *", Timezone: "UTC"}}}
|
||||
s := &Service{queue: q, config: config, backupMetrics: []model.BackupMetric{{RunID: "success", JobID: "job-1", Status: "success", FinishedAt: lastSuccess}}}
|
||||
status := s.DashboardStatus(now)
|
||||
if status.LastSuccessfulBackup == nil || status.LastSuccessfulBackup.RunID != "success" {
|
||||
t.Fatalf("last successful backup = %#v", status.LastSuccessfulBackup)
|
||||
}
|
||||
if status.LastFailedBackup == nil || status.LastFailedBackup.ID != "failed-backup" {
|
||||
t.Fatalf("last failed backup = %#v", status.LastFailedBackup)
|
||||
}
|
||||
if len(status.OverdueJobs) != 1 || status.OverdueJobs[0].JobID != "job-1" {
|
||||
t.Fatalf("overdue jobs = %#v", status.OverdueJobs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupMetricsOnlyCaptureCompletedSuccessfulBackupsOnce(t *testing.T) {
|
||||
st := store.New(t.TempDir())
|
||||
s := &Service{store: st}
|
||||
|
||||
@@ -79,6 +79,19 @@ type SnapshotBrowserPage struct {
|
||||
Generated time.Time `json:"generated"`
|
||||
}
|
||||
|
||||
type DashboardStatus struct {
|
||||
LastSuccessfulBackup *model.BackupMetric `json:"lastSuccessfulBackup,omitempty"`
|
||||
LastFailedBackup *model.Run `json:"lastFailedBackup,omitempty"`
|
||||
OverdueJobs []OverdueBackupJob `json:"overdueJobs"`
|
||||
}
|
||||
|
||||
type OverdueBackupJob struct {
|
||||
JobID string `json:"jobId"`
|
||||
Name string `json:"name"`
|
||||
ScheduledAt time.Time `json:"scheduledAt"`
|
||||
LastBackupAt *time.Time `json:"lastBackupAt,omitempty"`
|
||||
}
|
||||
|
||||
type snapshotTreeCache struct {
|
||||
generated time.Time
|
||||
total int
|
||||
@@ -127,6 +140,8 @@ func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runn
|
||||
s.queue = queue.New(s.execute, s.queuePersistence)
|
||||
if runs, err := st.LoadRuns(); err == nil {
|
||||
s.queue.RestoreHistory(runs)
|
||||
} else {
|
||||
log.Error("load run history", "error", err)
|
||||
}
|
||||
s.queuePersistence(s.queue.Snapshot())
|
||||
return s, nil
|
||||
@@ -418,6 +433,56 @@ func (s *Service) LastRun(jobID string) time.Time {
|
||||
return latest
|
||||
}
|
||||
|
||||
func (s *Service) DashboardStatus(now time.Time) DashboardStatus {
|
||||
status := DashboardStatus{OverdueJobs: []OverdueBackupJob{}}
|
||||
metrics := s.BackupMetrics()
|
||||
lastByJob := make(map[string]time.Time)
|
||||
for index := range metrics {
|
||||
metric := metrics[index]
|
||||
if metric.Status == "success" && (status.LastSuccessfulBackup == nil || metric.FinishedAt.After(status.LastSuccessfulBackup.FinishedAt)) {
|
||||
copy := metric
|
||||
status.LastSuccessfulBackup = ©
|
||||
}
|
||||
if metric.FinishedAt.After(lastByJob[metric.JobID]) {
|
||||
lastByJob[metric.JobID] = metric.FinishedAt
|
||||
}
|
||||
}
|
||||
runs := s.queue.Snapshot()
|
||||
activeJobs := make(map[string]bool)
|
||||
for index := range runs {
|
||||
run := runs[index]
|
||||
if run.TaskType != "backup" {
|
||||
continue
|
||||
}
|
||||
if run.Status == "queued" || run.Status == "running" || run.Status == "paused" {
|
||||
activeJobs[run.JobID] = true
|
||||
}
|
||||
if run.Status == "failed" && (status.LastFailedBackup == nil || run.CreatedAt.After(status.LastFailedBackup.CreatedAt)) {
|
||||
copy := run
|
||||
status.LastFailedBackup = ©
|
||||
}
|
||||
}
|
||||
for _, job := range s.Config().Jobs {
|
||||
if job.Type == model.JobRsync || !job.Enabled || job.Schedule.Cron == "" || activeJobs[job.ID] {
|
||||
continue
|
||||
}
|
||||
scheduledAt, err := scheduler.PreviousScheduled(job.Schedule.Cron, job.Schedule.Timezone, now, 370*24*time.Hour)
|
||||
if err != nil || scheduledAt.IsZero() || !lastByJob[job.ID].Before(scheduledAt) {
|
||||
continue
|
||||
}
|
||||
overdue := OverdueBackupJob{JobID: job.ID, Name: job.Name, ScheduledAt: scheduledAt}
|
||||
if last := lastByJob[job.ID]; !last.IsZero() {
|
||||
lastCopy := last
|
||||
overdue.LastBackupAt = &lastCopy
|
||||
}
|
||||
status.OverdueJobs = append(status.OverdueJobs, overdue)
|
||||
}
|
||||
sort.SliceStable(status.OverdueJobs, func(i, j int) bool {
|
||||
return status.OverdueJobs[i].ScheduledAt.Before(status.OverdueJobs[j].ScheduledAt)
|
||||
})
|
||||
return status
|
||||
}
|
||||
|
||||
func (s *Service) SaveConfig(config model.Config) error {
|
||||
config = model.NormalizeConfig(config)
|
||||
for _, job := range config.Jobs {
|
||||
|
||||
Reference in New Issue
Block a user