Harden backup and restore operations

internal/service/service.go

@@ -33,19 +33,25 @@ type SecretStore interface {
 }
 
 type Service struct {
-	store        *store.Store
-	secrets      SecretStore
-	restic       *restic.Runner
-	rsync        *rsync.Runner
-	mounts       *platform.MountManager
-	workloads    *platform.WorkloadManager
-	notifier     *notify.Sender
-	log          *slog.Logger
-	queue        *queue.Queue
-	mu           sync.RWMutex
-	config       model.Config
-	snapshotMu   sync.Mutex
-	snapshotTree map[string]snapshotTreeCache
+	store           *store.Store
+	secrets         SecretStore
+	restic          *restic.Runner
+	rsync           *rsync.Runner
+	mounts          *platform.MountManager
+	workloads       *platform.WorkloadManager
+	notifier        *notify.Sender
+	log             *slog.Logger
+	queue           *queue.Queue
+	mu              sync.RWMutex
+	config          model.Config
+	snapshotMu      sync.Mutex
+	snapshotTree    map[string]snapshotTreeCache
+	repositoryMu    sync.Mutex
+	repositoryLocks map[string]chan struct{}
+	persistRuns     chan []model.Run
+	persistStop     chan struct{}
+	persistDone     chan struct{}
+	persistStopOnce sync.Once
 }
 
 type SnapshotBrowserNode struct {
@@ -78,27 +84,46 @@ type snapshotTreeNode struct {
 	children map[string]*snapshotTreeNode
 }
 
+type snapshotTreeBuilder struct {
+	root  *snapshotTreeNode
+	nodes map[string]*snapshotTreeNode
+	total int
+}
+
 func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runner, mounts *platform.MountManager, workloads *platform.WorkloadManager, notifier *notify.Sender, log *slog.Logger) (*Service, error) {
 	config, err := st.LoadConfig()
 	if err != nil {
 		return nil, err
 	}
-	s := &Service{store: st, secrets: secrets, restic: rr, rsync: rs, mounts: mounts, workloads: workloads, notifier: notifier, log: log, config: config}
-	s.queue = queue.New(s.execute, func(runs []model.Run) {
-		if err := st.SaveRuns(runs); err != nil {
-			log.Error("save run history", "error", err)
-		}
-	})
+	s := &Service{
+		store: st, secrets: secrets, restic: rr, rsync: rs, mounts: mounts, workloads: workloads, notifier: notifier, log: log, config: config,
+		repositoryLocks: map[string]chan struct{}{}, persistRuns: make(chan []model.Run, 1), persistStop: make(chan struct{}), persistDone: make(chan struct{}),
+	}
+	go s.runPersistence()
+	s.queue = queue.New(s.execute, s.queuePersistence)
 	if runs, err := st.LoadRuns(); err == nil {
 		s.queue.RestoreHistory(runs)
 	}
+	s.queuePersistence(s.queue.Snapshot())
 	return s, nil
 }
 
 func (s *Service) RunQueue(ctx context.Context) { s.queue.Run(ctx) }
 func (s *Service) Stop()                        { s.queue.Stop() }
-func (s *Service) Config() model.Config         { s.mu.RLock(); defer s.mu.RUnlock(); return s.config }
-func (s *Service) Runs() []model.Run            { return s.queue.Snapshot() }
+func (s *Service) Wait(ctx context.Context) error {
+	if err := s.queue.Wait(ctx); err != nil {
+		return err
+	}
+	s.persistStopOnce.Do(func() { close(s.persistStop) })
+	select {
+	case <-s.persistDone:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+func (s *Service) Config() model.Config { s.mu.RLock(); defer s.mu.RUnlock(); return s.config }
+func (s *Service) Runs() []model.Run    { return s.queue.Snapshot() }
 func (s *Service) Logs() map[string]any {
 	return map[string]any{
 		"daemon": tailFile("/var/log/urbm.log", 1024*1024, 1000),
@@ -117,6 +142,123 @@ func runsWithLogs(runs []model.Run, limit int) []model.Run {
 	return result
 }
 
+func (s *Service) queuePersistence(runs []model.Run) {
+	select {
+	case s.persistRuns <- runs:
+		return
+	default:
+	}
+	select {
+	case <-s.persistRuns:
+	default:
+	}
+	select {
+	case s.persistRuns <- runs:
+	default:
+	}
+}
+
+func (s *Service) runPersistence() {
+	defer close(s.persistDone)
+	for {
+		select {
+		case runs := <-s.persistRuns:
+			s.saveRunState(runs)
+		case <-s.persistStop:
+			var latest []model.Run
+			for {
+				select {
+				case latest = <-s.persistRuns:
+				default:
+					if latest != nil {
+						s.saveRunState(latest)
+					}
+					return
+				}
+			}
+		}
+	}
+}
+
+func (s *Service) saveRunState(runs []model.Run) {
+	if err := s.store.SaveRuns(runs); err != nil {
+		s.log.Error("save run history", "error", err)
+	}
+	if err := s.persistRunLogs(runs); err != nil {
+		s.log.Error("save persistent run logs", "error", err)
+	}
+}
+
+func (s *Service) persistRunLogs(runs []model.Run) error {
+	dir := strings.TrimSpace(s.Config().Settings.PersistentLogDir)
+	if dir == "" {
+		return nil
+	}
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return err
+	}
+	wanted := make(map[string]struct{})
+	for _, run := range runs {
+		if run.FinishedAt == nil || len(run.LiveLog) == 0 {
+			continue
+		}
+		name := "urbm-" + safeLogName(run.ID) + ".log"
+		wanted[name] = struct{}{}
+		path := filepath.Join(dir, name)
+		if _, err := os.Stat(path); err == nil {
+			continue
+		}
+		if err := writeTextAtomic(path, strings.Join(run.LiveLog, "\n")+"\n"); err != nil {
+			return err
+		}
+	}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.HasPrefix(entry.Name(), "urbm-") || !strings.HasSuffix(entry.Name(), ".log") {
+			continue
+		}
+		if _, ok := wanted[entry.Name()]; !ok {
+			if err := os.Remove(filepath.Join(dir, entry.Name())); err != nil && !errors.Is(err, os.ErrNotExist) {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func safeLogName(value string) string {
+	return strings.Map(func(char rune) rune {
+		if char >= 'a' && char <= 'z' || char >= 'A' && char <= 'Z' || char >= '0' && char <= '9' || char == '-' || char == '_' {
+			return char
+		}
+		return '_'
+	}, value)
+}
+
+func writeTextAtomic(path, value string) error {
+	tmp, err := os.CreateTemp(filepath.Dir(path), ".urbm-log-*")
+	if err != nil {
+		return err
+	}
+	tmpPath := tmp.Name()
+	defer os.Remove(tmpPath)
+	if err := tmp.Chmod(0600); err != nil {
+		tmp.Close()
+		return err
+	}
+	if _, err := tmp.WriteString(value); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpPath, path)
+}
+
 func tailFile(path string, maxBytes int64, maxLines int) map[string]any {
 	data, err := os.ReadFile(path)
 	if err != nil {
@@ -193,8 +335,8 @@ func (s *Service) SaveConfig(config model.Config) error {
 	s.mu.Lock()
 	s.config = config
 	s.mu.Unlock()
-	s.restic.Binary = config.Settings.ResticPath
-	s.rsync.Binary = config.Settings.RsyncPath
+	s.restic.SetBinary(config.Settings.ResticPath)
+	s.rsync.SetBinary(config.Settings.RsyncPath)
 	return nil
 }
 
@@ -215,6 +357,11 @@ func (s *Service) ChangeRepositoryPassword(ctx context.Context, repoID, newPassw
 	if !ok {
 		return errors.New("validation: unknown repository")
 	}
+	unlock, err := s.lockRepository(ctx, repo)
+	if err != nil {
+		return err
+	}
+	defer unlock()
 	oldPassword, err := s.secrets.Get(repo.PasswordRef)
 	if err != nil {
 		return fmt.Errorf("authentication: load current repository password: %w", err)
@@ -260,6 +407,11 @@ func (s *Service) AdoptRepositoryPassword(ctx context.Context, repoID, password
 	if !ok {
 		return errors.New("validation: unknown repository")
 	}
+	unlock, err := s.lockRepository(ctx, repo)
+	if err != nil {
+		return err
+	}
+	defer unlock()
 	mounted, err := s.mounts.Prepare(ctx, repo)
 	if err != nil {
 		return err
@@ -313,18 +465,21 @@ func (s *Service) EnqueueMaintenance(repoID, action string) (model.Run, error) {
 }
 
 func (s *Service) EnqueueRestore(task model.RestoreTask) (model.Run, error) {
+	if err := model.ValidateRestoreTask(task); err != nil {
+		return model.Run{}, fmt.Errorf("validation: %w", err)
+	}
+	if _, ok := s.repository(task.RepositoryID); !ok {
+		return model.Run{}, errors.New("validation: unknown repository")
+	}
 	config := s.Config()
 	target, err := platform.ValidateRestoreTarget(task.Target, config.Settings.RestoreRoot, task.InPlace, task.Confirmed)
 	if err != nil {
 		return model.Run{}, err
 	}
 	task.Target = target
-	if task.ID == "" {
-		task.ID = newID()
-	}
+	task.ID = newID()
 	task.SchemaVersion = model.SchemaVersion
-	encoded := strings.Join(append([]string{task.RepositoryID, task.SnapshotID, task.Target, fmt.Sprint(task.InPlace)}, task.Includes...), "\x00")
-	run := model.Run{SchemaVersion: model.SchemaVersion, ID: task.ID, JobID: encoded, TaskType: "restore", Priority: 10, CreatedAt: time.Now().UTC()}
+	run := model.Run{SchemaVersion: model.SchemaVersion, ID: task.ID, JobID: task.RepositoryID, TaskType: "restore", Priority: 10, CreatedAt: time.Now().UTC(), Restore: &task}
 	return run, s.queue.Enqueue(run)
 }
 
@@ -375,6 +530,11 @@ func (s *Service) Snapshots(ctx context.Context, repoID string) ([]model.Snapsho
 	if !ok {
 		return nil, errors.New("validation: unknown repository")
 	}
+	unlock, err := s.lockRepository(ctx, repo)
+	if err != nil {
+		return nil, err
+	}
+	defer unlock()
 	mounted, err := s.mounts.Prepare(ctx, repo)
 	if err != nil {
 		return nil, err
@@ -388,6 +548,12 @@ func (s *Service) RepositoryStats(ctx context.Context) []model.RepositoryStats {
 	result := make([]model.RepositoryStats, 0, len(config.Repositories))
 	for _, repo := range config.Repositories {
 		item := model.RepositoryStats{RepositoryID: repo.ID, RepositoryName: repo.Name}
+		unlock, err := s.lockRepository(ctx, repo)
+		if err != nil {
+			item.Error = err.Error()
+			result = append(result, item)
+			break
+		}
 		mounted, err := s.mounts.Prepare(ctx, repo)
 		if err == nil {
 			var stored, files restic.Stats
@@ -406,6 +572,7 @@ func (s *Service) RepositoryStats(ctx context.Context) []model.RepositoryStats {
 				err = cleanupErr
 			}
 		}
+		unlock()
 		if err != nil {
 			item.Error = err.Error()
 		}
@@ -420,6 +587,9 @@ func (s *Service) RepositoryStats(ctx context.Context) []model.RepositoryStats {
 func (s *Service) SnapshotFiles(ctx context.Context, repoID, snapshot, path string) (SnapshotBrowserPage, error) {
 	started := time.Now()
 	s.log.Info("snapshot file listing started", "repoID", repoID, "snapshot", snapshot, "path", path)
+	if err := model.ValidateSnapshotID(snapshot); err != nil {
+		return SnapshotBrowserPage{}, fmt.Errorf("validation: %w", err)
+	}
 	repo, ok := s.repository(repoID)
 	if !ok {
 		return SnapshotBrowserPage{}, errors.New("validation: unknown repository")
@@ -431,6 +601,14 @@ func (s *Service) SnapshotFiles(ctx context.Context, repoID, snapshot, path stri
 		s.log.Info("snapshot file listing loaded from cache", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "items", len(page.Items), "total", page.Total, "durationMs", time.Since(started).Milliseconds())
 		return page, nil
 	}
+	unlock, err := s.lockRepository(ctx, repo)
+	if err != nil {
+		return SnapshotBrowserPage{}, err
+	}
+	defer unlock()
+	if page, ok := s.snapshotBrowserPage(key, normalizedPath); ok {
+		return page, nil
+	}
 
 	mounted, err := s.mounts.Prepare(ctx, repo)
 	if err != nil {
@@ -439,12 +617,24 @@ func (s *Service) SnapshotFiles(ctx context.Context, repoID, snapshot, path stri
 	}
 	defer s.mounts.Cleanup(context.Background(), mounted)
 
-	items, err := s.restic.List(ctx, mounted.Repository, snapshot, "")
+	const maxSnapshotEntries = 1_000_000
+	builder := newSnapshotTreeBuilder()
+	tooLarge := false
+	err = s.restic.WalkList(ctx, mounted.Repository, snapshot, "", func(item map[string]any) {
+		if builder.total >= maxSnapshotEntries {
+			tooLarge = true
+			return
+		}
+		builder.add(item)
+	})
 	if err != nil {
 		s.log.Warn("snapshot file listing failed", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "durationMs", time.Since(started).Milliseconds(), "error", err)
 		return SnapshotBrowserPage{}, err
 	}
-	cache := buildSnapshotTreeCache(items)
+	if tooLarge {
+		return SnapshotBrowserPage{}, fmt.Errorf("environment: snapshot contains more than %d entries; narrow the backup or use restic directly", maxSnapshotEntries)
+	}
+	cache := builder.cache()
 	s.storeSnapshotTree(key, cache)
 	page, _ := s.snapshotBrowserPage(key, normalizedPath)
 	s.log.Info("snapshot file listing loaded", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "items", len(page.Items), "total", cache.total, "durationMs", time.Since(started).Milliseconds())
@@ -480,53 +670,67 @@ func (s *Service) storeSnapshotTree(key string, cache snapshotTreeCache) {
 			delete(s.snapshotTree, cacheKey)
 		}
 	}
+	for len(s.snapshotTree) >= 3 {
+		oldestKey := ""
+		var oldest time.Time
+		for cacheKey, value := range s.snapshotTree {
+			if oldestKey == "" || value.generated.Before(oldest) {
+				oldestKey, oldest = cacheKey, value.generated
+			}
+		}
+		delete(s.snapshotTree, oldestKey)
+	}
 	s.snapshotTree[key] = cache
 }
 
 func buildSnapshotTreeCache(items []map[string]any) snapshotTreeCache {
-	root := &snapshotTreeNode{path: "/", name: "/", nodeType: "dir", children: map[string]*snapshotTreeNode{}}
-	nodes := map[string]*snapshotTreeNode{"/": root}
-	ensure := func(path string, nodeType string, size int64) *snapshotTreeNode {
-		return root
+	builder := newSnapshotTreeBuilder()
+	for _, item := range items {
+		builder.add(item)
 	}
-	ensure = func(path string, nodeType string, size int64) *snapshotTreeNode {
-		path = normalizeSnapshotPath(path)
-		if node, ok := nodes[path]; ok {
-			if nodeType != "" && nodeType != "dir" {
-				node.nodeType = nodeType
-				node.size = size
-			}
-			return node
+	return builder.cache()
+}
+
+func newSnapshotTreeBuilder() *snapshotTreeBuilder {
+	root := &snapshotTreeNode{path: "/", name: "/", nodeType: "dir", children: map[string]*snapshotTreeNode{}}
+	return &snapshotTreeBuilder{root: root, nodes: map[string]*snapshotTreeNode{"/": root}}
+}
+
+func (b *snapshotTreeBuilder) ensure(path string, nodeType string, size int64) *snapshotTreeNode {
+	path = normalizeSnapshotPath(path)
+	if node, ok := b.nodes[path]; ok {
+		if nodeType != "" && nodeType != "dir" {
+			node.nodeType, node.size = nodeType, size
 		}
-		parentPath := "/"
-		if path != "/" {
-			parentPath = filepath.Dir(path)
-			if parentPath == "." {
-				parentPath = "/"
-			}
-		}
-		parent := ensure(parentPath, "dir", 0)
-		node := &snapshotTreeNode{path: path, name: filepath.Base(path), nodeType: nodeType, size: size, children: map[string]*snapshotTreeNode{}}
-		if path == "/" {
-			node.name = "/"
-		}
-		nodes[path] = node
-		parent.children[path] = node
 		return node
 	}
-	for _, item := range items {
-		path := snapshotItemPath(item)
-		if path == "/" {
-			continue
-		}
-		nodeType := "file"
-		if value, _ := item["type"].(string); value == "dir" {
-			nodeType = "dir"
-		}
-		ensure(path, nodeType, snapshotItemSize(item))
+	parentPath := filepath.Dir(path)
+	if parentPath == "." {
+		parentPath = "/"
 	}
-	children := make(map[string][]SnapshotBrowserNode, len(nodes))
-	for _, node := range nodes {
+	parent := b.ensure(parentPath, "dir", 0)
+	node := &snapshotTreeNode{path: path, name: filepath.Base(path), nodeType: nodeType, size: size, children: map[string]*snapshotTreeNode{}}
+	b.nodes[path] = node
+	parent.children[path] = node
+	return node
+}
+
+func (b *snapshotTreeBuilder) add(item map[string]any) {
+	path := snapshotItemPath(item)
+	if path == "/" {
+		return
+	}
+	nodeType := "file"
+	if value, _ := item["type"].(string); value == "dir" {
+		nodeType = "dir"
+	}
+	b.ensure(path, nodeType, snapshotItemSize(item))
+	b.total++
+}
+
+func (b *snapshotTreeBuilder) cache() snapshotTreeCache {
+	children := make(map[string][]SnapshotBrowserNode, len(b.nodes))
+	for _, node := range b.nodes {
 		if len(node.children) == 0 {
 			continue
 		}
@@ -542,7 +746,7 @@ func buildSnapshotTreeCache(items []map[string]any) snapshotTreeCache {
 		})
 		children[node.path] = list
 	}
-	return snapshotTreeCache{generated: time.Now().UTC(), total: len(items), children: children}
+	return snapshotTreeCache{generated: time.Now().UTC(), total: b.total, children: children}
 }
 
 func snapshotItemPath(item map[string]any) string {
@@ -580,6 +784,11 @@ func (s *Service) TestRepository(ctx context.Context, repoID string, initialize
 	if !ok {
 		return errors.New("validation: unknown repository")
 	}
+	unlock, err := s.lockRepository(ctx, repo)
+	if err != nil {
+		return err
+	}
+	defer unlock()
 	mounted, err := s.mounts.Prepare(ctx, repo)
 	if err != nil {
 		return err
@@ -630,6 +839,11 @@ func (s *Service) UnlockRepository(ctx context.Context, repoID string) error {
 	if !ok {
 		return errors.New("validation: unknown repository")
 	}
+	unlock, err := s.lockRepository(ctx, repo)
+	if err != nil {
+		return err
+	}
+	defer unlock()
 	mounted, err := s.mounts.Prepare(ctx, repo)
 	if err != nil {
 		return err
@@ -643,6 +857,14 @@ func (s *Service) ForceUnlockRepository(ctx context.Context, repoID string) erro
 	if !ok {
 		return errors.New("validation: unknown repository")
 	}
+	if s.repositoryBusy(repoID) {
+		return errors.New("validation: repository has a queued, running, or paused task")
+	}
+	unlock, err := s.lockRepository(ctx, repo)
+	if err != nil {
+		return err
+	}
+	defer unlock()
 	mounted, err := s.mounts.Prepare(ctx, repo)
 	if err != nil {
 		return err
@@ -672,6 +894,11 @@ func (s *Service) execute(ctx context.Context, run model.Run) (result model.Run)
 	if !ok {
 		return failed(run, "validation", "repository no longer exists")
 	}
+	unlock, err := s.lockRepository(ctx, repo)
+	if err != nil {
+		return failedError(run, err)
+	}
+	defer unlock()
 	maintenanceRepoName = repo.Name
 	var pruneStatsBefore restic.Stats
 	pruneStatsBeforeOK := false
@@ -730,6 +957,9 @@ func (s *Service) executeRsync(ctx context.Context, run model.Run) (result model
 		return failed(run, "validation", "job no longer exists")
 	}
 	defer func() { result = s.finishJob(run, job, result) }()
+	if err := platform.ValidateRsyncPaths(job); err != nil {
+		return failedError(run, err)
+	}
 	live := run
 	appendLiveLog(&live, fmt.Sprintf("Rsync-Kopie wird vorbereitet: %d Quelle(n) nach %s", len(job.Sources), job.Rsync.Target))
 	s.updateActive(live)
@@ -819,6 +1049,11 @@ func (s *Service) executeBackup(ctx context.Context, run model.Run) (result mode
 	if !ok {
 		return failed(run, "validation", "repository no longer exists")
 	}
+	unlock, err := s.lockRepository(ctx, repo)
+	if err != nil {
+		return failedError(run, err)
+	}
+	defer unlock()
 	live := run
 	appendLiveLog(&live, "Backup wird vorbereitet")
 	s.updateActive(live)
@@ -1053,15 +1288,28 @@ func (s *Service) executeRestore(ctx context.Context, run model.Run) model.Run {
 	live := run
 	appendLiveLog(&live, "Restore wird vorbereitet")
 	s.updateActive(live)
-	parts := strings.Split(run.JobID, "\x00")
-	if len(parts) < 4 {
+	if run.Restore == nil {
 		return failed(run, "internal", "invalid restore payload")
 	}
-	repo, ok := s.repository(parts[0])
+	task := *run.Restore
+	if err := model.ValidateRestoreTask(task); err != nil {
+		return failed(run, "validation", err.Error())
+	}
+	config := s.Config()
+	target, err := platform.ValidateRestoreTarget(task.Target, config.Settings.RestoreRoot, task.InPlace, task.Confirmed)
+	if err != nil {
+		return failedError(run, err)
+	}
+	task.Target = target
+	repo, ok := s.repository(task.RepositoryID)
 	if !ok {
 		return failed(run, "validation", "repository no longer exists")
 	}
-	task := model.RestoreTask{SchemaVersion: model.SchemaVersion, ID: run.ID, RepositoryID: parts[0], SnapshotID: parts[1], Target: parts[2], InPlace: parts[3] == "true", Confirmed: true, Includes: parts[4:]}
+	unlock, err := s.lockRepository(ctx, repo)
+	if err != nil {
+		return failedError(run, err)
+	}
+	defer unlock()
 	if err := os.MkdirAll(task.Target, 0750); err != nil {
 		failedRun := failedError(run, err)
 		appendLiveLog(&live, "Restore-Ziel konnte nicht vorbereitet werden: "+err.Error())
@@ -1302,6 +1550,47 @@ func (s *Service) repository(id string) (model.Repository, bool) {
 	return model.Repository{}, false
 }
 
+func (s *Service) lockRepository(ctx context.Context, repo model.Repository) (func(), error) {
+	key := string(repo.Type) + ":" + filepath.Clean(repo.Location)
+	if repo.Mount != nil && repo.Mount.Managed {
+		key = "mount:" + filepath.Clean(repo.Mount.MountPoint)
+	}
+	s.repositoryMu.Lock()
+	lock := s.repositoryLocks[key]
+	if lock == nil {
+		lock = make(chan struct{}, 1)
+		s.repositoryLocks[key] = lock
+	}
+	s.repositoryMu.Unlock()
+	select {
+	case lock <- struct{}{}:
+		return func() { <-lock }, nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+func (s *Service) repositoryBusy(repoID string) bool {
+	for _, run := range s.queue.Snapshot() {
+		if run.Status != "queued" && run.Status != "running" && run.Status != "paused" {
+			continue
+		}
+		if run.TaskType == "restore" && run.Restore != nil && run.Restore.RepositoryID == repoID {
+			return true
+		}
+		if run.TaskType == "check" || run.TaskType == "prune" {
+			if run.JobID == repoID {
+				return true
+			}
+			continue
+		}
+		if job, ok := s.job(run.JobID); ok && job.RepositoryID == repoID {
+			return true
+		}
+	}
+	return false
+}
+
 func failed(run model.Run, code, message string) model.Run {
 	run.Status, run.ErrorCode, run.Message = "failed", code, message
 	return run