Harden backup and restore operations

internal/queue/queue_test.go

@@ -90,3 +90,33 @@ func TestPauseAndResumeActiveRun(t *testing.T) {
 	}
 	close(release)
 }
+
+func TestWaitDoesNotReturnBeforeCancelledHandlerCleanup(t *testing.T) {
+	started := make(chan struct{})
+	cleanup := make(chan struct{})
+	q := New(func(ctx context.Context, run model.Run) model.Run {
+		close(started)
+		<-ctx.Done()
+		close(cleanup)
+		run.Status = "cancelled"
+		return run
+	}, nil)
+	ctx, cancel := context.WithCancel(context.Background())
+	go q.Run(ctx)
+	if err := q.Enqueue(model.Run{ID: "run", JobID: "job"}); err != nil {
+		t.Fatal(err)
+	}
+	<-started
+	q.Stop()
+	waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
+	defer waitCancel()
+	if err := q.Wait(waitCtx); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case <-cleanup:
+	default:
+		t.Fatal("queue wait returned before handler cleanup")
+	}
+	cancel()
+}