Improve dashboard health and state recovery

internal/store/store.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"git.casaderoll.de/michael/urbm/internal/model"
 )
@@ -51,12 +52,20 @@ func (s *Store) SaveConfig(c model.Config) error {
 }
 
 func (s *Store) LoadRuns() ([]model.Run, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
+	s.mu.Lock()
+	defer s.mu.Unlock()
 	var runs []model.Run
 	if err := readJSON(s.runsPath(), &runs); errors.Is(err, os.ErrNotExist) {
 		return []model.Run{}, nil
 	} else if err != nil {
+		var decodeErr *jsonDecodeError
+		if errors.As(err, &decodeErr) {
+			quarantined, quarantineErr := quarantineFile(s.runsPath())
+			if quarantineErr != nil {
+				return nil, fmt.Errorf("%w; quarantine failed: %v", err, quarantineErr)
+			}
+			return nil, fmt.Errorf("%w; corrupt file moved to %s", err, quarantined)
+		}
 		return nil, err
 	}
 	return runs, nil
@@ -72,8 +81,8 @@ func (s *Store) SaveRuns(runs []model.Run) error {
 }
 
 func (s *Store) LoadBackupMetrics() ([]model.BackupMetric, bool, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
+	s.mu.Lock()
+	defer s.mu.Unlock()
 	path := s.backupMetricsPath()
 	b, err := os.ReadFile(path)
 	legacySource := false
@@ -95,7 +104,7 @@ func (s *Store) LoadBackupMetrics() ([]model.BackupMetric, bool, error) {
 	if b[0] == '[' {
 		var metrics []model.BackupMetric
 		if err := json.Unmarshal(b, &metrics); err != nil {
-			return nil, true, fmt.Errorf("decode %s: %w", path, err)
+			return nil, true, quarantineDecodeError(path, err)
 		}
 		return metrics, true, nil
 	}
@@ -108,7 +117,7 @@ func (s *Store) LoadBackupMetrics() ([]model.BackupMetric, bool, error) {
 		}
 		var metric model.BackupMetric
 		if err := json.Unmarshal(line, &metric); err != nil {
-			return nil, legacySource, fmt.Errorf("decode %s line %d: %w", path, index+1, err)
+			return nil, legacySource, quarantineDecodeError(path, fmt.Errorf("line %d: %w", index+1, err))
 		}
 		metrics = append(metrics, metric)
 	}
@@ -169,11 +178,36 @@ func readJSON(path string, target any) error {
 		return err
 	}
 	if err := json.Unmarshal(b, target); err != nil {
-		return fmt.Errorf("decode %s: %w", path, err)
+		return &jsonDecodeError{path: path, err: err}
 	}
 	return nil
 }
 
+type jsonDecodeError struct {
+	path string
+	err  error
+}
+
+func (e *jsonDecodeError) Error() string { return fmt.Sprintf("decode %s: %v", e.path, e.err) }
+func (e *jsonDecodeError) Unwrap() error { return e.err }
+
+func quarantineDecodeError(path string, decodeErr error) error {
+	quarantined, err := quarantineFile(path)
+	wrapped := &jsonDecodeError{path: path, err: decodeErr}
+	if err != nil {
+		return fmt.Errorf("%w; quarantine failed: %v", wrapped, err)
+	}
+	return fmt.Errorf("%w; corrupt file moved to %s", wrapped, quarantined)
+}
+
+func quarantineFile(path string) (string, error) {
+	quarantined := path + ".corrupt-" + time.Now().UTC().Format("20060102-150405.000000000")
+	if err := os.Rename(path, quarantined); err != nil {
+		return "", err
+	}
+	return quarantined, nil
+}
+
 func writeJSONAtomic(path string, value any, mode os.FileMode) error {
 	b, err := json.MarshalIndent(value, "", "  ")
 	if err != nil {

internal/store/store_test.go

@@ -2,6 +2,7 @@ package store
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -30,6 +31,49 @@ func TestStoreInitializesAndPersists(t *testing.T) {
 	}
 }
 
+func TestCorruptRunHistoryIsQuarantinedBeforeReplacement(t *testing.T) {
+	dir := t.TempDir()
+	s := New(dir)
+	if err := os.WriteFile(s.runsPath(), []byte(`{"broken":`), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := s.LoadRuns(); err == nil {
+		t.Fatal("corrupt run history loaded without error")
+	}
+	if _, err := os.Stat(s.runsPath()); !os.IsNotExist(err) {
+		t.Fatalf("corrupt runs file was not moved: %v", err)
+	}
+	matches, err := filepath.Glob(s.runsPath() + ".corrupt-*")
+	if err != nil || len(matches) != 1 {
+		t.Fatalf("quarantined run files = %#v, %v", matches, err)
+	}
+	if err := s.SaveRuns([]model.Run{}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(matches[0]); err != nil {
+		t.Fatalf("quarantined run file was overwritten: %v", err)
+	}
+}
+
+func TestCorruptBackupMetricsAreQuarantined(t *testing.T) {
+	dir := t.TempDir()
+	s := New(dir)
+	if err := os.WriteFile(s.backupMetricsPath(), []byte("{not-json}\n"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := s.LoadBackupMetrics(); err == nil {
+		t.Fatal("corrupt backup metrics loaded without error")
+	}
+	matches, err := filepath.Glob(s.backupMetricsPath() + ".corrupt-*")
+	if err != nil || len(matches) != 1 {
+		t.Fatalf("quarantined metric files = %#v, %v", matches, err)
+	}
+	contents, err := os.ReadFile(matches[0])
+	if err != nil || string(contents) != "{not-json}\n" {
+		t.Fatalf("quarantined metric contents = %q, %v", contents, err)
+	}
+}
+
 func TestBackupMetricsPersistSeparatelyFromRuns(t *testing.T) {
 	s := New(t.TempDir())
 	if err := s.Init(); err != nil {