michael/URBM
0
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Release 2026.09.27.r002: clarify rsync dry runs and expand run diagnostics

Browse Source
This commit is contained in:
Mikei386 committed 2026-09-27 16:23:01 +02:00
1 parent 106f9c9387
commit 74a69b1580
13 files changed
+451 -46
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/rsync/rsync.go
+37 -19
View File
@@ -40,16 +40,19 @@ type Progress struct {
Bytes int64
Files int64
CurrentFile string
Change string
}
type Summary struct {
Bytes int64
Files int64
Bytes int64
Files int64
Deleted int64
}
var (
progressPattern = regexp.MustCompile(`^\s*([0-9,]+)\s+([0-9]+)%`)
filesPattern = regexp.MustCompile(`^Number of regular files transferred:\s*([0-9,]+)`)
deletedPattern = regexp.MustCompile(`^Number of deleted files:\s*([0-9,]+)`)
bytesPattern = regexp.MustCompile(`^Total transferred file size:\s*([0-9,]+) bytes`)
)
@@ -88,17 +91,12 @@ func (r *Runner) Run(ctx context.Context, runID string, job model.Job, callback
done := make(chan struct{})
go func() {
scanOutput(stdout, func(line string) {
if value, ok := parseCount(filesPattern, line); ok {
summary.Files = value
}
if value, ok := parseCount(bytesPattern, line); ok {
summary.Bytes = value
}
summary.parseLine(line)
if callback != nil {
if progress, ok := parseProgress(line); ok {
callback(progress)
} else if strings.HasPrefix(line, "FILE|") {
callback(Progress{CurrentFile: strings.TrimPrefix(line, "FILE|")})
} else if change, ok := parseChange(line); ok {
callback(change)
}
}
})
@@ -106,9 +104,10 @@ func (r *Runner) Run(ctx context.Context, runID string, job model.Job, callback
}()
stderrDone := make(chan []byte, 1)
go func() { stderrDone <- drainLimited(stderr, 1024*1024) }()
err = cmd.Wait()
// Drain both pipes before Wait closes them, including the final statistics.
<-done
errBytes := <-stderrDone
err = cmd.Wait()
if err != nil {
if errors.Is(ctx.Err(), context.Canceled) {
return Summary{}, fmt.Errorf("cancelled: %w", ctx.Err())
@@ -163,20 +162,16 @@ func (r *Runner) Estimate(ctx context.Context, runID string, job model.Job) (Sum
done := make(chan struct{})
go func() {
scanOutput(stdout, func(line string) {
if value, ok := parseCount(filesPattern, line); ok {
summary.Files = value
}
if value, ok := parseCount(bytesPattern, line); ok {
summary.Bytes = value
}
summary.parseLine(line)
})
close(done)
}()
stderrDone := make(chan []byte, 1)
go func() { stderrDone <- drainLimited(stderr, 1024*1024) }()
err = cmd.Wait()
// Drain both pipes before Wait closes them, including the final statistics.
<-done
errBytes := <-stderrDone
err = cmd.Wait()
if err != nil {
if errors.Is(ctx.Err(), context.Canceled) {
return Summary{}, fmt.Errorf("cancelled: %w", ctx.Err())
@@ -188,7 +183,7 @@ func (r *Runner) Estimate(ctx context.Context, runID string, job model.Job) (Sum
func Arguments(job model.Job) []string {
options := job.Rsync
args := []string{"--recursive", "--human-readable", "--info=progress2", "--stats", "--partial", "--out-format=FILE|%n"}
args := []string{"--recursive", "--info=progress2", "--stats", "--partial", "--outbuf=L", "--out-format=CHANGE|%i|%n%L"}
if !options.Overwrite {
args = append(args, "--ignore-existing")
}
@@ -322,3 +317,26 @@ func parseCount(pattern *regexp.Regexp, line string) (int64, bool) {
value, err := strconv.ParseInt(strings.ReplaceAll(match[1], ",", ""), 10, 64)
return value, err == nil
}
func (s *Summary) parseLine(line string) {
if value, ok := parseCount(filesPattern, line); ok {
s.Files = value
}
if value, ok := parseCount(bytesPattern, line); ok {
s.Bytes = value
}
if value, ok := parseCount(deletedPattern, line); ok {
s.Deleted = value
}
}
func parseChange(line string) (Progress, bool) {
if !strings.HasPrefix(line, "CHANGE|") {
return Progress{}, false
}
fields := strings.SplitN(strings.TrimPrefix(line, "CHANGE|"), "|", 2)
if len(fields) != 2 || len(fields[0]) != 11 {
return Progress{}, false
}
return Progress{Change: fields[0], CurrentFile: fields[1]}, true
}
internal/rsync/rsync_test.go
+139
View File
@@ -1,7 +1,11 @@
package rsync
import (
"context"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"git.casaderoll.de/michael/urbm/internal/model"
@@ -31,3 +35,138 @@ func TestArrayDiskMirrorKeepsDiskDirectory(t *testing.T) {
t.Fatalf("mirror arguments = %v", args)
}
}
func TestMachineReadableArgumentsAndSummary(t *testing.T) {
args := Arguments(model.Job{})
if slices.Contains(args, "--human-readable") {
t.Fatal("human readable sizes break byte parsing")
}
if !slices.Contains(args, "--out-format=CHANGE|%i|%n%L") {
t.Fatal(args)
}
var summary Summary
for _, line := range []string{"Number of regular files transferred: 1,376", "Number of deleted files: 1,214 (reg: 1,200, dir: 14)", "Total transferred file size: 1,234,567,890 bytes"} {
summary.parseLine(line)
}
if summary.Files != 1376 || summary.Deleted != 1214 || summary.Bytes != 1234567890 {
t.Fatalf("summary = %+v", summary)
}
}
func TestParseChangePreservesNames(t *testing.T) {
for _, code := range []string{">f+++++++++", ">f.st......", ".d..t......", "*deleting "} {
got, ok := parseChange("CHANGE|" + code + "|disk1/a | b.txt")
if !ok || got.Change != code || got.CurrentFile != "disk1/a | b.txt" {
t.Fatalf("change = %+v, %v", got, ok)
}
}
}
func TestRunnerReadsFinalStats(t *testing.T) {
binary := filepath.Join(t.TempDir(), "rsync")
script := "#!/bin/sh\nprintf '%s\\n' 'CHANGE|>f+++++++++|disk1/new.txt' 'Number of regular files transferred: 1,376' 'Number of deleted files: 1,214 (reg: 1214)' 'Total transferred file size: 1,234,567,890 bytes'\n"
if err := os.WriteFile(binary, []byte(script), 0700); err != nil {
t.Fatal(err)
}
runner := Runner{Binary: binary}
for _, estimate := range []bool{false, true} {
var summary Summary
var err error
var changes []string
if estimate {
summary, err = runner.Estimate(context.Background(), "test", model.Job{})
} else {
summary, err = runner.Run(context.Background(), "test", model.Job{}, func(p Progress) {
if p.CurrentFile != "" {
changes = append(changes, p.CurrentFile)
}
})
}
if err != nil || summary.Files != 1376 || summary.Deleted != 1214 || summary.Bytes != 1234567890 {
t.Fatalf("estimate=%v summary=%+v err=%v", estimate, summary, err)
}
if !estimate && strings.Join(changes, ",") != "disk1/new.txt" {
t.Fatal(changes)
}
}
}
// Opt in with URBM_TEST_RSYNC pointing to rsync 3.x (Unraid's supported version).
func TestRealRsyncDryRunAndCopy(t *testing.T) {
binary := os.Getenv("URBM_TEST_RSYNC")
if binary == "" {
t.Skip("set URBM_TEST_RSYNC to test with rsync 3.x")
}
dir := t.TempDir()
source, target := filepath.Join(dir, "disk1"), filepath.Join(dir, "USB")
destination := filepath.Join(target, "disk1")
for _, path := range []string{source, filepath.Join(destination, "old-dir")} {
if err := os.MkdirAll(path, 0700); err != nil {
t.Fatal(err)
}
}
for path, data := range map[string]string{
filepath.Join(source, "new"): "new", filepath.Join(source, "changed"): "updated",
filepath.Join(source, "same"): "same", filepath.Join(destination, "same"): "same",
filepath.Join(destination, "changed"): "old", filepath.Join(destination, "obsolete"): "delete",
filepath.Join(destination, "old-dir", "file"): "delete",
} {
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
t.Fatal(err)
}
}
if err := os.Symlink("obsolete", filepath.Join(destination, "old-link")); err != nil {
t.Fatal(err)
}
runner := Runner{Binary: binary}
job := model.Job{Sources: []model.Source{{Path: source}}, Rsync: model.RsyncOptions{Target: target, DryRun: true, Overwrite: true, Delete: true, Checksum: true, PreserveLinks: true}}
var changes []Progress
summary, err := runner.Run(context.Background(), "test", job, func(p Progress) {
if p.Change != "" {
changes = append(changes, p)
}
})
if err != nil || summary.Files != 2 || summary.Bytes != 10 || summary.Deleted != 4 {
t.Fatalf("dry run: %+v %v", summary, err)
}
if len(changes) < 6 {
t.Fatalf("missing change details: %+v", changes)
}
if _, err := os.Stat(filepath.Join(destination, "new")); !os.IsNotExist(err) {
t.Fatal("dry run wrote a file")
}
if _, err := os.Stat(filepath.Join(destination, "obsolete")); err != nil {
t.Fatal("dry run deleted a file")
}
estimate, err := runner.Estimate(context.Background(), "test", job)
if err != nil || estimate != summary {
t.Fatalf("estimate differs: %+v %v", estimate, err)
}
job.Rsync.DryRun = false
copied, err := runner.Run(context.Background(), "test", job, nil)
if err != nil || copied != summary {
t.Fatalf("copy differs: %+v %v", copied, err)
}
for _, name := range []string{"new", "changed", "same"} {
got, err := os.ReadFile(filepath.Join(destination, name))
if err != nil {
t.Fatal(err)
}
want, err := os.ReadFile(filepath.Join(source, name))
if err != nil {
t.Fatal(err)
}
if string(got) != string(want) {
t.Fatalf("different contents: %s", name)
}
}
for _, name := range []string{"obsolete", "old-dir", "old-link"} {
if _, err := os.Lstat(filepath.Join(destination, name)); !os.IsNotExist(err) {
t.Fatalf("not deleted: %s", name)
}
}
identical, err := runner.Estimate(context.Background(), "test", job)
if err != nil || identical.Files != 0 || identical.Bytes != 0 || identical.Deleted != 0 {
t.Fatalf("identical: %+v %v", identical, err)
}
}