Improve dashboard health and state recovery

internal/service/service.go

@@ -79,6 +79,19 @@ type SnapshotBrowserPage struct {
 	Generated time.Time             `json:"generated"`
 }
 
+type DashboardStatus struct {
+	LastSuccessfulBackup *model.BackupMetric `json:"lastSuccessfulBackup,omitempty"`
+	LastFailedBackup     *model.Run          `json:"lastFailedBackup,omitempty"`
+	OverdueJobs          []OverdueBackupJob  `json:"overdueJobs"`
+}
+
+type OverdueBackupJob struct {
+	JobID        string     `json:"jobId"`
+	Name         string     `json:"name"`
+	ScheduledAt  time.Time  `json:"scheduledAt"`
+	LastBackupAt *time.Time `json:"lastBackupAt,omitempty"`
+}
+
 type snapshotTreeCache struct {
 	generated time.Time
 	total     int
@@ -127,6 +140,8 @@ func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runn
 	s.queue = queue.New(s.execute, s.queuePersistence)
 	if runs, err := st.LoadRuns(); err == nil {
 		s.queue.RestoreHistory(runs)
+	} else {
+		log.Error("load run history", "error", err)
 	}
 	s.queuePersistence(s.queue.Snapshot())
 	return s, nil
@@ -418,6 +433,56 @@ func (s *Service) LastRun(jobID string) time.Time {
 	return latest
 }
 
+func (s *Service) DashboardStatus(now time.Time) DashboardStatus {
+	status := DashboardStatus{OverdueJobs: []OverdueBackupJob{}}
+	metrics := s.BackupMetrics()
+	lastByJob := make(map[string]time.Time)
+	for index := range metrics {
+		metric := metrics[index]
+		if metric.Status == "success" && (status.LastSuccessfulBackup == nil || metric.FinishedAt.After(status.LastSuccessfulBackup.FinishedAt)) {
+			copy := metric
+			status.LastSuccessfulBackup = &copy
+		}
+		if metric.FinishedAt.After(lastByJob[metric.JobID]) {
+			lastByJob[metric.JobID] = metric.FinishedAt
+		}
+	}
+	runs := s.queue.Snapshot()
+	activeJobs := make(map[string]bool)
+	for index := range runs {
+		run := runs[index]
+		if run.TaskType != "backup" {
+			continue
+		}
+		if run.Status == "queued" || run.Status == "running" || run.Status == "paused" {
+			activeJobs[run.JobID] = true
+		}
+		if run.Status == "failed" && (status.LastFailedBackup == nil || run.CreatedAt.After(status.LastFailedBackup.CreatedAt)) {
+			copy := run
+			status.LastFailedBackup = &copy
+		}
+	}
+	for _, job := range s.Config().Jobs {
+		if job.Type == model.JobRsync || !job.Enabled || job.Schedule.Cron == "" || activeJobs[job.ID] {
+			continue
+		}
+		scheduledAt, err := scheduler.PreviousScheduled(job.Schedule.Cron, job.Schedule.Timezone, now, 370*24*time.Hour)
+		if err != nil || scheduledAt.IsZero() || !lastByJob[job.ID].Before(scheduledAt) {
+			continue
+		}
+		overdue := OverdueBackupJob{JobID: job.ID, Name: job.Name, ScheduledAt: scheduledAt}
+		if last := lastByJob[job.ID]; !last.IsZero() {
+			lastCopy := last
+			overdue.LastBackupAt = &lastCopy
+		}
+		status.OverdueJobs = append(status.OverdueJobs, overdue)
+	}
+	sort.SliceStable(status.OverdueJobs, func(i, j int) bool {
+		return status.OverdueJobs[i].ScheduledAt.Before(status.OverdueJobs[j].ScheduledAt)
+	})
+	return status
+}
+
 func (s *Service) SaveConfig(config model.Config) error {
 	config = model.NormalizeConfig(config)
 	for _, job := range config.Jobs {