Fix backup check notifications and workload recovery; release 2026.09.21.r001

2026-09-21 20:36:31 +02:00
parent 8f75a04db6
commit 5009dc202b
14 changed files with 295 additions and 38 deletions
@@ -0,0 +1,120 @@
+package platform
+
+import (
+	"context"
+	"git.casaderoll.de/michael/urbm/internal/model"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func fakeDocker(t *testing.T, body string) string {
+	t.Helper()
+	dir := t.TempDir()
+	path := filepath.Join(dir, "docker")
+	if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body), 0700); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
+	return dir
+}
+
+func TestPreparePreservesStoppedContainersAndIncludesVolumes(t *testing.T) {
+	dir := fakeDocker(t, `case "$1" in
+ inspect)
+ if [ "$2" = --format ]; then
+   if [ "$4" = off ]; then echo '{"Running":false}'; else echo '{"Running":true}'; fi
+ else echo '[{"Mounts":[{"Type":"bind","Source":"/appdata"},{"Type":"volume","Source":"/docker/volumes/db/_data"},{"Type":"tmpfs","Source":""}]}]'; fi;;
+ stop|start) echo "$1 $2" >> "$ACTIONS";;
+ esac
+`)
+	actions := filepath.Join(dir, "actions")
+	t.Setenv("ACTIONS", actions)
+	w := &WorkloadManager{RuntimeDir: dir}
+	job := model.Job{ID: "test", Type: model.JobDocker, Sources: []model.Source{{WorkloadID: "off"}, {WorkloadID: "on"}}}
+	job.Consistency.Mode = "stop"
+	p, err := w.Prepare(context.Background(), job)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p.Stopped) != 1 || p.Stopped[0].WorkloadID != "on" {
+		t.Fatalf("stopped: %+v", p.Stopped)
+	}
+	if !strings.Contains(strings.Join(p.Sources, "\n"), "/docker/volumes/db/_data") {
+		t.Fatal("named volume omitted")
+	}
+	if err = w.Cleanup(context.Background(), p); err != nil {
+		t.Fatal(err)
+	}
+	data, _ := os.ReadFile(actions)
+	if strings.Contains(string(data), "start off") || !strings.Contains(string(data), "start on") {
+		t.Fatalf("actions: %s", data)
+	}
+}
+
+func TestRecoveryContinuesAfterFailureAndCancellation(t *testing.T) {
+	dir := fakeDocker(t, `echo "$1 $2" >> "$ACTIONS"
+ if [ "$2" = broken ]; then exit 1; fi
+`)
+	actions := filepath.Join(dir, "actions")
+	t.Setenv("ACTIONS", actions)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	w := &WorkloadManager{}
+	err := w.Cleanup(ctx, Prepared{Kind: model.JobDocker, Stopped: []model.Source{{WorkloadID: "healthy"}, {WorkloadID: "broken"}}})
+	if err == nil || !strings.Contains(err.Error(), "broken") {
+		t.Fatalf("error: %v", err)
+	}
+	data, _ := os.ReadFile(actions)
+	if !strings.Contains(string(data), "start healthy") {
+		t.Fatalf("recovery aborted: %s", data)
+	}
+}
+
+func TestPrepareRollsBackPreviouslyStoppedContainers(t *testing.T) {
+	dir := fakeDocker(t, `case "$1" in
+ inspect)
+ if [ "$2" = bad ]; then exit 1; fi
+ if [ "$2" = --format ]; then echo '{"Running":true}'; else echo '[{"Mounts":[]}]'; fi;;
+ stop|start) echo "$1 $2" >> "$ACTIONS";;
+ esac
+`)
+	actions := filepath.Join(dir, "actions")
+	t.Setenv("ACTIONS", actions)
+	w := &WorkloadManager{RuntimeDir: dir}
+	job := model.Job{ID: "test", Type: model.JobDocker, Sources: []model.Source{{WorkloadID: "first"}, {WorkloadID: "bad"}}}
+	job.Consistency.Mode = "stop"
+	if _, err := w.Prepare(context.Background(), job); err == nil {
+		t.Fatal("expected failure")
+	}
+	data, _ := os.ReadFile(actions)
+	if !strings.Contains(string(data), "start first") {
+		t.Fatalf("missing rollback: %s", data)
+	}
+}
+
+func TestSlowRecoveryDoesNotExhaustOtherStarts(t *testing.T) {
+	dir := fakeDocker(t, `if [ "$2" = slow ]; then sleep 6; fi
+ echo "$2" >> "$ACTIONS"
+`)
+	actions := filepath.Join(dir, "actions")
+	t.Setenv("ACTIONS", actions)
+	err := (&WorkloadManager{}).Cleanup(context.Background(), Prepared{Kind: model.JobDocker, Stopped: []model.Source{{WorkloadID: "next"}, {WorkloadID: "slow"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, _ := os.ReadFile(actions)
+	if string(data) != "slow\nnext\n" {
+		t.Fatalf("incomplete recovery: %s", data)
+	}
+}
+
+func TestPersistentMountWithoutPathFails(t *testing.T) {
+	dir := fakeDocker(t, `echo '[{"Mounts":[{"Type":"volume","Source":""}]}]'
+`)
+	w := &WorkloadManager{}
+	if _, err := w.captureMetadata(context.Background(), model.JobDocker, "db", dir); err == nil {
+		t.Fatal("missing volume path silently omitted")
+	}
+}