Add scheduled rsync copy jobs

This commit is contained in:
Mikei386
2026-06-15 17:15:27 +02:00
parent 2abfc8eb26
commit 6c795fd97d
15 changed files with 524 additions and 59 deletions
+43 -18
View File
@@ -1,6 +1,9 @@
package model
import "time"
import (
"strings"
"time"
)
const SchemaVersion = 1
@@ -12,6 +15,7 @@ const (
JobDocker JobType = "docker"
JobVM JobType = "vm"
JobFlash JobType = "flash"
JobRsync JobType = "rsync"
)
type RepositoryType string
@@ -33,6 +37,7 @@ type Config struct {
type Settings struct {
ResticPath string `json:"resticPath"`
RsyncPath string `json:"rsyncPath"`
RestoreRoot string `json:"restoreRoot"`
PersistentLogDir string `json:"persistentLogDir,omitempty"`
ShowLiveLog bool `json:"showLiveLog"`
@@ -42,23 +47,39 @@ type Settings struct {
}
type Job struct {
SchemaVersion int `json:"schemaVersion"`
ID string `json:"id"`
Name string `json:"name"`
Type JobType `json:"type"`
Enabled bool `json:"enabled"`
RepositoryID string `json:"repositoryId"`
Sources []Source `json:"sources"`
Schedule Schedule `json:"schedule"`
Consistency Consistency `json:"consistency"`
Compression string `json:"compression"`
CPUCores int `json:"cpuCores,omitempty"`
Excludes []string `json:"excludes,omitempty"`
Tags []string `json:"tags,omitempty"`
FlashImage bool `json:"flashImage,omitempty"`
Retention Retention `json:"retention"`
NotifyOn []string `json:"notifyOn,omitempty"`
ShutdownSecs int `json:"shutdownTimeoutSeconds"`
SchemaVersion int `json:"schemaVersion"`
ID string `json:"id"`
Name string `json:"name"`
Type JobType `json:"type"`
Enabled bool `json:"enabled"`
RepositoryID string `json:"repositoryId"`
Sources []Source `json:"sources"`
Schedule Schedule `json:"schedule"`
Consistency Consistency `json:"consistency"`
Compression string `json:"compression"`
CPUCores int `json:"cpuCores,omitempty"`
Excludes []string `json:"excludes,omitempty"`
Tags []string `json:"tags,omitempty"`
FlashImage bool `json:"flashImage,omitempty"`
Retention Retention `json:"retention"`
NotifyOn []string `json:"notifyOn,omitempty"`
ShutdownSecs int `json:"shutdownTimeoutSeconds"`
Rsync RsyncOptions `json:"rsync,omitempty"`
}
type RsyncOptions struct {
Target string `json:"target"`
Overwrite bool `json:"overwrite"`
Delete bool `json:"delete"`
PreservePermissions bool `json:"preservePermissions"`
PreserveOwner bool `json:"preserveOwner"`
PreserveGroup bool `json:"preserveGroup"`
PreserveTimes bool `json:"preserveTimes"`
PreserveLinks bool `json:"preserveLinks"`
PreserveACLs bool `json:"preserveAcls"`
PreserveXattrs bool `json:"preserveXattrs"`
Checksum bool `json:"checksum"`
DryRun bool `json:"dryRun"`
}
type Source struct {
@@ -180,6 +201,7 @@ func DefaultConfig() Config {
Notifications: []NotificationTarget{{SchemaVersion: SchemaVersion, ID: "unraid", Name: "Unraid", Type: "unraid", Enabled: true, Events: []string{"success", "warning", "failed"}}},
Settings: Settings{
ResticPath: "/usr/local/libexec/urbm/restic",
RsyncPath: "/usr/bin/rsync",
RestoreRoot: "/mnt/user/urbm-restores",
CatchUpWindowHrs: 24,
CheckCron: "0 3 1 * *",
@@ -189,6 +211,9 @@ func DefaultConfig() Config {
}
func NormalizeConfig(c Config) Config {
if strings.TrimSpace(c.Settings.RsyncPath) == "" {
c.Settings.RsyncPath = "/usr/bin/rsync"
}
notifications := make([]NotificationTarget, 0, len(c.Notifications)+1)
hasUnraid := false
for _, target := range c.Notifications {
+41 -8
View File
@@ -34,8 +34,10 @@ func ValidateConfig(c Config) error {
return fmt.Errorf("duplicate job id %q", job.ID)
}
jobs[job.ID] = struct{}{}
if _, exists := repos[job.RepositoryID]; !exists {
return fmt.Errorf("job %q references unknown repository %q", job.ID, job.RepositoryID)
if job.Type != JobRsync {
if _, exists := repos[job.RepositoryID]; !exists {
return fmt.Errorf("job %q references unknown repository %q", job.ID, job.RepositoryID)
}
}
}
notifications := make(map[string]struct{}, len(c.Notifications))
@@ -90,11 +92,11 @@ func ValidateJob(j Job) error {
if j.SchemaVersion != SchemaVersion || !idPattern.MatchString(j.ID) {
return errors.New("invalid schemaVersion or id")
}
if strings.TrimSpace(j.Name) == "" || strings.TrimSpace(j.RepositoryID) == "" {
return errors.New("name and repositoryId are required")
if strings.TrimSpace(j.Name) == "" {
return errors.New("name is required")
}
switch j.Type {
case JobShare, JobAppdata, JobDocker, JobVM, JobFlash:
case JobShare, JobAppdata, JobDocker, JobVM, JobFlash, JobRsync:
default:
return fmt.Errorf("unsupported job type %q", j.Type)
}
@@ -119,6 +121,17 @@ func ValidateJob(j Job) error {
}
}
}
for _, exclude := range j.Excludes {
if strings.ContainsAny(exclude, "\x00\r\n") {
return errors.New("exclude contains forbidden control characters")
}
}
if j.Type == JobRsync {
return validateRsyncJob(j)
}
if strings.TrimSpace(j.RepositoryID) == "" {
return errors.New("repositoryId is required")
}
if j.Compression != "auto" && j.Compression != "off" && j.Compression != "max" {
return errors.New("compression must be auto, off, or max")
}
@@ -134,14 +147,34 @@ func ValidateJob(j Job) error {
if j.Retention.KeepWithinDays == 0 && j.Retention.Daily == 0 && j.Retention.Weekly == 0 && j.Retention.Monthly == 0 {
return errors.New("retention must keep at least one age or calendar policy")
}
for _, exclude := range j.Excludes {
if strings.ContainsAny(exclude, "\x00\r\n") {
return errors.New("exclude contains forbidden control characters")
return nil
}
func validateRsyncJob(j Job) error {
target := filepath.Clean(strings.TrimSpace(j.Rsync.Target))
if !filepath.IsAbs(target) {
return errors.New("rsync target must be absolute")
}
if !pathWithin(target, "/mnt/user") && !pathWithin(target, "/mnt/disks") && !pathWithin(target, "/mnt/remotes") {
return errors.New("rsync target must be below /mnt/user, /mnt/disks, or /mnt/remotes")
}
for _, source := range j.Sources {
if source.WorkloadID != "" || source.Path == "" {
return errors.New("rsync jobs require filesystem paths")
}
cleanSource := filepath.Clean(source.Path)
if pathWithin(target, cleanSource) || pathWithin(cleanSource, target) {
return fmt.Errorf("rsync source %q and target %q must not contain each other", cleanSource, target)
}
}
return nil
}
func pathWithin(path, root string) bool {
rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path))
return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
}
func ValidateRepository(r Repository) error {
if r.SchemaVersion != SchemaVersion || !idPattern.MatchString(r.ID) {
return errors.New("invalid schemaVersion or id")
+15
View File
@@ -91,3 +91,18 @@ func TestBackupCPUCoresRange(t *testing.T) {
t.Fatal("excessive CPU limit accepted")
}
}
func TestRsyncJobDoesNotRequireRepository(t *testing.T) {
c := DefaultConfig()
c.Jobs = []Job{{SchemaVersion: 1, ID: "rsync-job", Name: "Mirror", Type: JobRsync, Enabled: true, Sources: []Source{{Path: "/mnt/user/data"}}, Rsync: RsyncOptions{Target: "/mnt/remotes/backup", Overwrite: true}}}
if err := ValidateConfig(c); err != nil {
t.Fatalf("valid rsync job rejected: %v", err)
}
}
func TestRsyncJobRejectsOverlappingPaths(t *testing.T) {
job := Job{SchemaVersion: 1, ID: "rsync-job", Name: "Mirror", Type: JobRsync, Sources: []Source{{Path: "/mnt/user/data"}}, Rsync: RsyncOptions{Target: "/mnt/user/data/copy"}}
if err := ValidateJob(job); err == nil {
t.Fatal("rsync target inside source accepted")
}
}
+228
View File
@@ -0,0 +1,228 @@
package rsync
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"sync"
"syscall"
"git.casaderoll.de/michael/urbm/internal/model"
)
type Runner struct {
Binary string
mu sync.Mutex
processes map[string]*os.Process
}
type Progress struct {
Percent float64
Bytes int64
Files int64
CurrentFile string
}
type Summary struct {
Bytes int64
Files int64
}
var (
progressPattern = regexp.MustCompile(`^\s*([0-9,]+)\s+([0-9]+)%`)
filesPattern = regexp.MustCompile(`^Number of regular files transferred:\s*([0-9,]+)`)
bytesPattern = regexp.MustCompile(`^Total transferred file size:\s*([0-9,]+) bytes`)
)
func (r *Runner) Run(ctx context.Context, runID string, job model.Job, callback func(Progress)) (Summary, error) {
args := Arguments(job)
cmd := exec.CommandContext(ctx, r.Binary, args...)
cmd.Env = append(os.Environ(), "LC_ALL=C", "LANG=C")
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
cmd.Cancel = func() error {
if cmd.Process == nil {
return os.ErrProcessDone
}
if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) {
return err
}
return nil
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return Summary{}, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return Summary{}, err
}
if err := cmd.Start(); err != nil {
if errors.Is(err, exec.ErrNotFound) {
return Summary{}, fmt.Errorf("environment: rsync binary not found: %w", err)
}
return Summary{}, err
}
unregister := r.register(runID, cmd.Process)
defer unregister()
var summary Summary
done := make(chan struct{})
go func() {
scanOutput(stdout, func(line string) {
if value, ok := parseCount(filesPattern, line); ok {
summary.Files = value
}
if value, ok := parseCount(bytesPattern, line); ok {
summary.Bytes = value
}
if callback != nil {
if progress, ok := parseProgress(line); ok {
callback(progress)
} else if strings.HasPrefix(line, "FILE|") {
callback(Progress{CurrentFile: strings.TrimPrefix(line, "FILE|")})
}
}
})
close(done)
}()
errBytes, _ := io.ReadAll(io.LimitReader(stderr, 1024*1024))
err = cmd.Wait()
<-done
if err != nil {
if errors.Is(ctx.Err(), context.Canceled) {
return Summary{}, fmt.Errorf("cancelled: %w", ctx.Err())
}
message := strings.TrimSpace(string(errBytes))
if message == "" {
message = err.Error()
}
return Summary{}, fmt.Errorf("rsync: %s", message)
}
return summary, nil
}
func Arguments(job model.Job) []string {
options := job.Rsync
args := []string{"--recursive", "--human-readable", "--info=progress2", "--stats", "--partial", "--out-format=FILE|%n"}
if !options.Overwrite {
args = append(args, "--ignore-existing")
}
if options.Delete {
args = append(args, "--delete-delay")
}
if options.PreservePermissions {
args = append(args, "--perms")
}
if options.PreserveOwner {
args = append(args, "--owner")
}
if options.PreserveGroup {
args = append(args, "--group")
}
if options.PreserveTimes {
args = append(args, "--times")
}
if options.PreserveLinks {
args = append(args, "--links")
}
if options.PreserveACLs {
args = append(args, "--acls")
}
if options.PreserveXattrs {
args = append(args, "--xattrs")
}
if options.Checksum {
args = append(args, "--checksum")
}
if options.DryRun {
args = append(args, "--dry-run")
}
for _, exclude := range job.Excludes {
args = append(args, "--exclude", exclude)
}
args = append(args, "--")
for _, source := range job.Sources {
args = append(args, source.Path)
}
args = append(args, strings.TrimRight(options.Target, "/")+"/")
return args
}
func (r *Runner) Pause(runID string) error { return r.signal(runID, syscall.SIGSTOP) }
func (r *Runner) Resume(runID string) error { return r.signal(runID, syscall.SIGCONT) }
func (r *Runner) signal(runID string, signal syscall.Signal) error {
r.mu.Lock()
process := r.processes[runID]
r.mu.Unlock()
if process == nil {
return errors.New("rsync process is not active yet")
}
if err := syscall.Kill(-process.Pid, signal); err != nil {
return fmt.Errorf("signal rsync process group: %w", err)
}
return nil
}
func (r *Runner) register(runID string, process *os.Process) func() {
r.mu.Lock()
if r.processes == nil {
r.processes = make(map[string]*os.Process)
}
r.processes[runID] = process
r.mu.Unlock()
return func() {
r.mu.Lock()
delete(r.processes, runID)
r.mu.Unlock()
}
}
func scanOutput(reader io.Reader, callback func(string)) {
scanner := bufio.NewScanner(reader)
scanner.Split(splitLinesAndCarriageReturns)
scanner.Buffer(make([]byte, 64*1024), 2*1024*1024)
for scanner.Scan() {
if line := strings.TrimSpace(scanner.Text()); line != "" {
callback(line)
}
}
}
func splitLinesAndCarriageReturns(data []byte, atEOF bool) (advance int, token []byte, err error) {
for i, value := range data {
if value == '\n' || value == '\r' {
return i + 1, data[:i], nil
}
}
if atEOF && len(data) > 0 {
return len(data), data, nil
}
return 0, nil, nil
}
func parseProgress(line string) (Progress, bool) {
match := progressPattern.FindStringSubmatch(line)
if len(match) != 3 {
return Progress{}, false
}
bytes, _ := strconv.ParseInt(strings.ReplaceAll(match[1], ",", ""), 10, 64)
percent, _ := strconv.ParseFloat(match[2], 64)
return Progress{Bytes: bytes, Percent: percent}, true
}
func parseCount(pattern *regexp.Regexp, line string) (int64, bool) {
match := pattern.FindStringSubmatch(line)
if len(match) != 2 {
return 0, false
}
value, err := strconv.ParseInt(strings.ReplaceAll(match[1], ",", ""), 10, 64)
return value, err == nil
}
+25
View File
@@ -0,0 +1,25 @@
package rsync
import (
"slices"
"testing"
"git.casaderoll.de/michael/urbm/internal/model"
)
func TestArgumentsUseExplicitSafeOptions(t *testing.T) {
job := model.Job{Sources: []model.Source{{Path: "/mnt/user/data"}}, Excludes: []string{"*.tmp"}, Rsync: model.RsyncOptions{Target: "/mnt/remotes/backup", Delete: true, PreservePermissions: true}}
args := Arguments(job)
for _, expected := range []string{"--recursive", "--delete-delay", "--perms", "--exclude", "*.tmp", "--", "/mnt/user/data", "/mnt/remotes/backup/"} {
if !slices.Contains(args, expected) {
t.Fatalf("missing %q in %#v", expected, args)
}
}
}
func TestParseProgress(t *testing.T) {
progress, ok := parseProgress(" 1,234,567 42% 12.34MB/s 0:00:05")
if !ok || progress.Bytes != 1234567 || progress.Percent != 42 {
t.Fatalf("progress = %#v, %v", progress, ok)
}
}
+96 -8
View File
@@ -18,6 +18,7 @@ import (
"git.casaderoll.de/michael/urbm/internal/platform"
"git.casaderoll.de/michael/urbm/internal/queue"
"git.casaderoll.de/michael/urbm/internal/restic"
"git.casaderoll.de/michael/urbm/internal/rsync"
"git.casaderoll.de/michael/urbm/internal/scheduler"
"git.casaderoll.de/michael/urbm/internal/store"
)
@@ -32,6 +33,7 @@ type Service struct {
store *store.Store
secrets SecretStore
restic *restic.Runner
rsync *rsync.Runner
mounts *platform.MountManager
workloads *platform.WorkloadManager
notifier *notify.Sender
@@ -41,12 +43,12 @@ type Service struct {
config model.Config
}
func New(st *store.Store, secrets SecretStore, rr *restic.Runner, mounts *platform.MountManager, workloads *platform.WorkloadManager, notifier *notify.Sender, log *slog.Logger) (*Service, error) {
func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runner, mounts *platform.MountManager, workloads *platform.WorkloadManager, notifier *notify.Sender, log *slog.Logger) (*Service, error) {
config, err := st.LoadConfig()
if err != nil {
return nil, err
}
s := &Service{store: st, secrets: secrets, restic: rr, mounts: mounts, workloads: workloads, notifier: notifier, log: log, config: config}
s := &Service{store: st, secrets: secrets, restic: rr, rsync: rs, mounts: mounts, workloads: workloads, notifier: notifier, log: log, config: config}
s.queue = queue.New(s.execute, func(runs []model.Run) {
if err := st.SaveRuns(runs); err != nil {
log.Error("save run history", "error", err)
@@ -112,6 +114,7 @@ func (s *Service) SaveConfig(config model.Config) error {
s.config = config
s.mu.Unlock()
s.restic.Binary = config.Settings.ResticPath
s.rsync.Binary = config.Settings.RsyncPath
return nil
}
@@ -129,7 +132,11 @@ func (s *Service) EnqueueJob(jobID string, priority int) (model.Run, error) {
if !ok {
return model.Run{}, errors.New("validation: unknown job")
}
run := model.Run{SchemaVersion: model.SchemaVersion, ID: newID(), JobID: job.ID, TaskType: "backup", Priority: priority, CreatedAt: time.Now().UTC()}
taskType := "backup"
if job.Type == model.JobRsync {
taskType = "rsync"
}
run := model.Run{SchemaVersion: model.SchemaVersion, ID: newID(), JobID: job.ID, TaskType: taskType, Priority: priority, CreatedAt: time.Now().UTC()}
return run, s.queue.Enqueue(run)
}
@@ -169,18 +176,27 @@ func (s *Service) Cancel(runID string) bool { return s.queue.Cancel(runID) }
func (s *Service) ClearRunHistory() int { return s.queue.ClearHistory() }
func (s *Service) Pause(runID string) error {
if err := s.restic.Pause(runID); err != nil {
runner := s.restic.Pause
resume := s.restic.Resume
if s.runTaskType(runID) == "rsync" {
runner, resume = s.rsync.Pause, s.rsync.Resume
}
if err := runner(runID); err != nil {
return fmt.Errorf("pause: %w", err)
}
if !s.queue.SetPaused(runID, true) {
_ = s.restic.Resume(runID)
_ = resume(runID)
return errors.New("pause: run is not currently running")
}
return nil
}
func (s *Service) Resume(runID string) error {
if err := s.restic.Resume(runID); err != nil {
runner := s.restic.Resume
if s.runTaskType(runID) == "rsync" {
runner = s.rsync.Resume
}
if err := runner(runID); err != nil {
return fmt.Errorf("resume: %w", err)
}
if !s.queue.SetPaused(runID, false) {
@@ -189,6 +205,15 @@ func (s *Service) Resume(runID string) error {
return nil
}
func (s *Service) runTaskType(runID string) string {
for _, run := range s.queue.Snapshot() {
if run.ID == runID {
return run.TaskType
}
}
return ""
}
func (s *Service) Snapshots(ctx context.Context, repoID string) ([]model.Snapshot, error) {
repo, ok := s.repository(repoID)
if !ok {
@@ -317,6 +342,9 @@ func (s *Service) execute(ctx context.Context, run model.Run) model.Run {
if run.TaskType == "backup" {
return s.executeBackup(ctx, run)
}
if run.TaskType == "rsync" {
return s.executeRsync(ctx, run)
}
if run.TaskType == "restore" {
return s.executeRestore(ctx, run)
}
@@ -354,6 +382,66 @@ func (s *Service) execute(ctx context.Context, run model.Run) model.Run {
return run
}
func (s *Service) executeRsync(ctx context.Context, run model.Run) (result model.Run) {
job, ok := s.job(run.JobID)
if !ok {
return failed(run, "validation", "job no longer exists")
}
defer func() { result = s.finishJob(run, job, result) }()
live := run
appendLiveLog(&live, fmt.Sprintf("Rsync-Kopie wird vorbereitet: %d Quelle(n) nach %s", len(job.Sources), job.Rsync.Target))
s.updateActive(live)
for _, source := range job.Sources {
if _, err := os.Stat(source.Path); err != nil {
return failed(run, "source", "source unavailable: "+source.Path)
}
}
if info, err := os.Stat(job.Rsync.Target); err != nil || !info.IsDir() {
if err == nil {
err = errors.New("target is not a directory")
}
return failed(run, "environment", "rsync target unavailable: "+err.Error())
}
started := time.Now()
lastLoggedPercent := -10
summary, err := s.rsync.Run(ctx, run.ID, job, func(progress rsync.Progress) {
if progress.Percent > 0 || progress.Bytes > 0 {
live.ProgressPercent = progress.Percent
live.ProgressBytes = progress.Bytes
if progress.Percent > 0 {
live.ProgressTotal = int64(float64(progress.Bytes) / (progress.Percent / 100))
}
if elapsed := time.Since(started).Seconds(); elapsed > 0 {
live.BytesPerSecond = float64(progress.Bytes) / elapsed
}
percent := int(progress.Percent)
if percent >= lastLoggedPercent+10 {
appendLiveLog(&live, fmt.Sprintf("Rsync-Fortschritt: %.0f%%, %s übertragen", progress.Percent, formatBytes(progress.Bytes)))
lastLoggedPercent = percent - percent%10
}
}
if progress.CurrentFile != "" {
live.CurrentFile = progress.CurrentFile
}
s.updateActive(live)
})
if err != nil {
appendLiveLog(&live, "Rsync-Kopie fehlgeschlagen: "+err.Error())
failedRun := failedError(run, err)
failedRun.LiveLog = live.LiveLog
return failedRun
}
run.Status, run.Message = "success", "rsync completed"
run.BytesAdded, run.BytesProcessed = summary.Bytes, summary.Bytes
run.FilesNew, run.FilesProcessed = summary.Files, summary.Files
run.ProgressPercent, run.ProgressBytes, run.ProgressTotal = 100, summary.Bytes, summary.Bytes
run.ProgressFiles, run.ProgressFileTotal = summary.Files, summary.Files
run.BytesPerSecond, run.CurrentFile = live.BytesPerSecond, live.CurrentFile
appendLiveLog(&live, fmt.Sprintf("Rsync-Kopie abgeschlossen: %d Dateien, %s", summary.Files, formatBytes(summary.Bytes)))
run.LiveLog = live.LiveLog
return run
}
func (s *Service) executeBackup(ctx context.Context, run model.Run) (result model.Run) {
job, ok := s.job(run.JobID)
if !ok {
@@ -577,7 +665,7 @@ func notificationMessage(run model.Run, now time.Time) string {
lines = append(lines, "Dauer: "+formatDuration(duration))
}
}
if run.TaskType == "backup" && (run.Status == "success" || run.Status == "warning") {
if (run.TaskType == "backup" || run.TaskType == "rsync") && (run.Status == "success" || run.Status == "warning") {
lines = append(lines,
"Neu gespeichert: "+formatBytes(run.BytesAdded),
fmt.Sprintf("Neue Dateien: %d", run.FilesNew),
@@ -589,7 +677,7 @@ func notificationMessage(run model.Run, now time.Time) string {
lines = append(lines, "Snapshot: "+run.SnapshotID)
}
}
if run.Message != "" && run.Message != "backup completed" {
if run.Message != "" && run.Message != "backup completed" && run.Message != "rsync completed" {
lines = append(lines, "Details: "+run.Message)
}
return strings.Join(lines, "\n")