Improve dashboard health and state recovery

internal/api/server.go

@@ -40,6 +40,7 @@ func New(socket string, svc *service.Service, log *slog.Logger) *Server {
 	mux.HandleFunc("GET /v1/runs", s.runs)
 	mux.HandleFunc("DELETE /v1/runs", s.clearRuns)
 	mux.HandleFunc("GET /v1/backup-metrics", s.backupMetrics)
+	mux.HandleFunc("GET /v1/dashboard-status", s.dashboardStatus)
 	mux.HandleFunc("GET /v1/logs", s.logs)
 	mux.HandleFunc("GET /v1/filesystem/directories", s.directories)
 	mux.HandleFunc("GET /v1/workloads/{kind}", s.workloads)
@@ -133,6 +134,9 @@ func (s *Server) runs(w http.ResponseWriter, _ *http.Request) { writeJSON(w, 200
 func (s *Server) backupMetrics(w http.ResponseWriter, _ *http.Request) {
 	writeJSON(w, 200, s.service.BackupMetrics())
 }
+func (s *Server) dashboardStatus(w http.ResponseWriter, _ *http.Request) {
+	writeJSON(w, 200, s.service.DashboardStatus(time.Now()))
+}
 func (s *Server) clearRuns(w http.ResponseWriter, _ *http.Request) {
 	writeJSON(w, 200, map[string]int{"cleared": s.service.ClearRunHistory()})
 }
@@ -364,7 +368,7 @@ func shouldLogRequest(r *http.Request, status int, duration time.Duration) bool
 		return true
 	}
 	switch r.URL.Path {
-	case "/v1/runs", "/v1/backup-metrics", "/v1/health", "/v1/logs":
+	case "/v1/runs", "/v1/backup-metrics", "/v1/dashboard-status", "/v1/health", "/v1/logs":
 		return false
 	default:
 		return true

internal/scheduler/cron_test.go

@@ -44,6 +44,18 @@ func TestPreviousWithinCatchUpWindow(t *testing.T) {
 	}
 }
 
+func TestPreviousScheduledUsesJobTimezone(t *testing.T) {
+	now := time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC)
+	want := time.Date(2026, 7, 13, 0, 0, 0, 0, time.UTC)
+	got, err := PreviousScheduled("0 2 * * *", "Europe/Berlin", now, 24*time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !got.Equal(want) {
+		t.Fatalf("previous scheduled = %v, want %v", got, want)
+	}
+}
+
 func TestCronUsesOrForRestrictedMonthDayAndWeekday(t *testing.T) {
 	expr, err := Parse("0 2 1 * 1")
 	if err != nil {

internal/scheduler/scheduler.go

@@ -14,6 +14,21 @@ type Enqueue func(model.Job) error
 type EnqueueMaintenance func(string, string) (model.Run, error)
 type LastRun func(string) time.Time
 
+func PreviousScheduled(spec, timezone string, now time.Time, window time.Duration) (time.Time, error) {
+	expression, err := Parse(spec)
+	if err != nil {
+		return time.Time{}, err
+	}
+	loc := time.Local
+	if timezone != "" {
+		loc, err = time.LoadLocation(timezone)
+		if err != nil {
+			return time.Time{}, err
+		}
+	}
+	return previous(expression, now.In(loc), window).UTC(), nil
+}
+
 type Scheduler struct {
 	config      ConfigProvider
 	enqueue     Enqueue

internal/service/safety_test.go

@@ -79,6 +79,30 @@ func TestLastRunUsesDurableMetricAfterActivityClear(t *testing.T) {
 	}
 }
 
+func TestDashboardStatusUsesOnlyBackupsAndFindsOverdueJobs(t *testing.T) {
+	now := time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC)
+	lastSuccess := now.Add(-30 * time.Hour)
+	failedAt := now.Add(-2 * time.Hour)
+	q := queue.New(nil, nil)
+	q.RestoreHistory([]model.Run{
+		{ID: "failed-backup", JobID: "job-1", TaskType: "backup", Status: "failed", CreatedAt: failedAt},
+		{ID: "successful-check", JobID: "repo-1", TaskType: "check", Status: "success", CreatedAt: now.Add(-time.Hour)},
+	})
+	config := model.DefaultConfig()
+	config.Jobs = []model.Job{{ID: "job-1", Name: "Daily", Type: model.JobShare, Enabled: true, Schedule: model.Schedule{Cron: "0 2 * * *", Timezone: "UTC"}}}
+	s := &Service{queue: q, config: config, backupMetrics: []model.BackupMetric{{RunID: "success", JobID: "job-1", Status: "success", FinishedAt: lastSuccess}}}
+	status := s.DashboardStatus(now)
+	if status.LastSuccessfulBackup == nil || status.LastSuccessfulBackup.RunID != "success" {
+		t.Fatalf("last successful backup = %#v", status.LastSuccessfulBackup)
+	}
+	if status.LastFailedBackup == nil || status.LastFailedBackup.ID != "failed-backup" {
+		t.Fatalf("last failed backup = %#v", status.LastFailedBackup)
+	}
+	if len(status.OverdueJobs) != 1 || status.OverdueJobs[0].JobID != "job-1" {
+		t.Fatalf("overdue jobs = %#v", status.OverdueJobs)
+	}
+}
+
 func TestBackupMetricsOnlyCaptureCompletedSuccessfulBackupsOnce(t *testing.T) {
 	st := store.New(t.TempDir())
 	s := &Service{store: st}

internal/service/service.go

@@ -79,6 +79,19 @@ type SnapshotBrowserPage struct {
 	Generated time.Time             `json:"generated"`
 }
 
+type DashboardStatus struct {
+	LastSuccessfulBackup *model.BackupMetric `json:"lastSuccessfulBackup,omitempty"`
+	LastFailedBackup     *model.Run          `json:"lastFailedBackup,omitempty"`
+	OverdueJobs          []OverdueBackupJob  `json:"overdueJobs"`
+}
+
+type OverdueBackupJob struct {
+	JobID        string     `json:"jobId"`
+	Name         string     `json:"name"`
+	ScheduledAt  time.Time  `json:"scheduledAt"`
+	LastBackupAt *time.Time `json:"lastBackupAt,omitempty"`
+}
+
 type snapshotTreeCache struct {
 	generated time.Time
 	total     int
@@ -127,6 +140,8 @@ func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runn
 	s.queue = queue.New(s.execute, s.queuePersistence)
 	if runs, err := st.LoadRuns(); err == nil {
 		s.queue.RestoreHistory(runs)
+	} else {
+		log.Error("load run history", "error", err)
 	}
 	s.queuePersistence(s.queue.Snapshot())
 	return s, nil
@@ -418,6 +433,56 @@ func (s *Service) LastRun(jobID string) time.Time {
 	return latest
 }
 
+func (s *Service) DashboardStatus(now time.Time) DashboardStatus {
+	status := DashboardStatus{OverdueJobs: []OverdueBackupJob{}}
+	metrics := s.BackupMetrics()
+	lastByJob := make(map[string]time.Time)
+	for index := range metrics {
+		metric := metrics[index]
+		if metric.Status == "success" && (status.LastSuccessfulBackup == nil || metric.FinishedAt.After(status.LastSuccessfulBackup.FinishedAt)) {
+			copy := metric
+			status.LastSuccessfulBackup = &copy
+		}
+		if metric.FinishedAt.After(lastByJob[metric.JobID]) {
+			lastByJob[metric.JobID] = metric.FinishedAt
+		}
+	}
+	runs := s.queue.Snapshot()
+	activeJobs := make(map[string]bool)
+	for index := range runs {
+		run := runs[index]
+		if run.TaskType != "backup" {
+			continue
+		}
+		if run.Status == "queued" || run.Status == "running" || run.Status == "paused" {
+			activeJobs[run.JobID] = true
+		}
+		if run.Status == "failed" && (status.LastFailedBackup == nil || run.CreatedAt.After(status.LastFailedBackup.CreatedAt)) {
+			copy := run
+			status.LastFailedBackup = &copy
+		}
+	}
+	for _, job := range s.Config().Jobs {
+		if job.Type == model.JobRsync || !job.Enabled || job.Schedule.Cron == "" || activeJobs[job.ID] {
+			continue
+		}
+		scheduledAt, err := scheduler.PreviousScheduled(job.Schedule.Cron, job.Schedule.Timezone, now, 370*24*time.Hour)
+		if err != nil || scheduledAt.IsZero() || !lastByJob[job.ID].Before(scheduledAt) {
+			continue
+		}
+		overdue := OverdueBackupJob{JobID: job.ID, Name: job.Name, ScheduledAt: scheduledAt}
+		if last := lastByJob[job.ID]; !last.IsZero() {
+			lastCopy := last
+			overdue.LastBackupAt = &lastCopy
+		}
+		status.OverdueJobs = append(status.OverdueJobs, overdue)
+	}
+	sort.SliceStable(status.OverdueJobs, func(i, j int) bool {
+		return status.OverdueJobs[i].ScheduledAt.Before(status.OverdueJobs[j].ScheduledAt)
+	})
+	return status
+}
+
 func (s *Service) SaveConfig(config model.Config) error {
 	config = model.NormalizeConfig(config)
 	for _, job := range config.Jobs {

internal/store/store.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"git.casaderoll.de/michael/urbm/internal/model"
 )
@@ -51,12 +52,20 @@ func (s *Store) SaveConfig(c model.Config) error {
 }
 
 func (s *Store) LoadRuns() ([]model.Run, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
+	s.mu.Lock()
+	defer s.mu.Unlock()
 	var runs []model.Run
 	if err := readJSON(s.runsPath(), &runs); errors.Is(err, os.ErrNotExist) {
 		return []model.Run{}, nil
 	} else if err != nil {
+		var decodeErr *jsonDecodeError
+		if errors.As(err, &decodeErr) {
+			quarantined, quarantineErr := quarantineFile(s.runsPath())
+			if quarantineErr != nil {
+				return nil, fmt.Errorf("%w; quarantine failed: %v", err, quarantineErr)
+			}
+			return nil, fmt.Errorf("%w; corrupt file moved to %s", err, quarantined)
+		}
 		return nil, err
 	}
 	return runs, nil
@@ -72,8 +81,8 @@ func (s *Store) SaveRuns(runs []model.Run) error {
 }
 
 func (s *Store) LoadBackupMetrics() ([]model.BackupMetric, bool, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
+	s.mu.Lock()
+	defer s.mu.Unlock()
 	path := s.backupMetricsPath()
 	b, err := os.ReadFile(path)
 	legacySource := false
@@ -95,7 +104,7 @@ func (s *Store) LoadBackupMetrics() ([]model.BackupMetric, bool, error) {
 	if b[0] == '[' {
 		var metrics []model.BackupMetric
 		if err := json.Unmarshal(b, &metrics); err != nil {
-			return nil, true, fmt.Errorf("decode %s: %w", path, err)
+			return nil, true, quarantineDecodeError(path, err)
 		}
 		return metrics, true, nil
 	}
@@ -108,7 +117,7 @@ func (s *Store) LoadBackupMetrics() ([]model.BackupMetric, bool, error) {
 		}
 		var metric model.BackupMetric
 		if err := json.Unmarshal(line, &metric); err != nil {
-			return nil, legacySource, fmt.Errorf("decode %s line %d: %w", path, index+1, err)
+			return nil, legacySource, quarantineDecodeError(path, fmt.Errorf("line %d: %w", index+1, err))
 		}
 		metrics = append(metrics, metric)
 	}
@@ -169,11 +178,36 @@ func readJSON(path string, target any) error {
 		return err
 	}
 	if err := json.Unmarshal(b, target); err != nil {
-		return fmt.Errorf("decode %s: %w", path, err)
+		return &jsonDecodeError{path: path, err: err}
 	}
 	return nil
 }
 
+type jsonDecodeError struct {
+	path string
+	err  error
+}
+
+func (e *jsonDecodeError) Error() string { return fmt.Sprintf("decode %s: %v", e.path, e.err) }
+func (e *jsonDecodeError) Unwrap() error { return e.err }
+
+func quarantineDecodeError(path string, decodeErr error) error {
+	quarantined, err := quarantineFile(path)
+	wrapped := &jsonDecodeError{path: path, err: decodeErr}
+	if err != nil {
+		return fmt.Errorf("%w; quarantine failed: %v", wrapped, err)
+	}
+	return fmt.Errorf("%w; corrupt file moved to %s", wrapped, quarantined)
+}
+
+func quarantineFile(path string) (string, error) {
+	quarantined := path + ".corrupt-" + time.Now().UTC().Format("20060102-150405.000000000")
+	if err := os.Rename(path, quarantined); err != nil {
+		return "", err
+	}
+	return quarantined, nil
+}
+
 func writeJSONAtomic(path string, value any, mode os.FileMode) error {
 	b, err := json.MarshalIndent(value, "", "  ")
 	if err != nil {

internal/store/store_test.go

@@ -2,6 +2,7 @@ package store
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -30,6 +31,49 @@ func TestStoreInitializesAndPersists(t *testing.T) {
 	}
 }
 
+func TestCorruptRunHistoryIsQuarantinedBeforeReplacement(t *testing.T) {
+	dir := t.TempDir()
+	s := New(dir)
+	if err := os.WriteFile(s.runsPath(), []byte(`{"broken":`), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := s.LoadRuns(); err == nil {
+		t.Fatal("corrupt run history loaded without error")
+	}
+	if _, err := os.Stat(s.runsPath()); !os.IsNotExist(err) {
+		t.Fatalf("corrupt runs file was not moved: %v", err)
+	}
+	matches, err := filepath.Glob(s.runsPath() + ".corrupt-*")
+	if err != nil || len(matches) != 1 {
+		t.Fatalf("quarantined run files = %#v, %v", matches, err)
+	}
+	if err := s.SaveRuns([]model.Run{}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(matches[0]); err != nil {
+		t.Fatalf("quarantined run file was overwritten: %v", err)
+	}
+}
+
+func TestCorruptBackupMetricsAreQuarantined(t *testing.T) {
+	dir := t.TempDir()
+	s := New(dir)
+	if err := os.WriteFile(s.backupMetricsPath(), []byte("{not-json}\n"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := s.LoadBackupMetrics(); err == nil {
+		t.Fatal("corrupt backup metrics loaded without error")
+	}
+	matches, err := filepath.Glob(s.backupMetricsPath() + ".corrupt-*")
+	if err != nil || len(matches) != 1 {
+		t.Fatalf("quarantined metric files = %#v, %v", matches, err)
+	}
+	contents, err := os.ReadFile(matches[0])
+	if err != nil || string(contents) != "{not-json}\n" {
+		t.Fatalf("quarantined metric contents = %q, %v", contents, err)
+	}
+}
+
 func TestBackupMetricsPersistSeparatelyFromRuns(t *testing.T) {
 	s := New(t.TempDir())
 	if err := s.Init(); err != nil {