Add automatic best segment detection

2026-06-04 12:17:49 +02:00
parent 9edf3eaf8a
commit 8f57d1f429
3 changed files with 232 additions and 18 deletions
@@ -175,6 +175,9 @@ struct ContentView: View {
                .disabled(processor.state.isRunning)

                Section("Gesicht") {
+                    Toggle("Bestes Segment automatisch finden", isOn: $settings.bestSegmentDetectionEnabled)
+                        .help("Analysiert das Originalvideo vor dem Schneiden und sucht das Fenster, in dem Gesicht und beide Augen am besten sichtbar sind. Wenn nichts Brauchbares gefunden wird, nutzt GrowthLapse die normale Mitte/Start-Offset-Logik.")
+
                    Toggle("Gesichtsgröße normalisieren", isOn: $settings.faceNormalizationEnabled)
                        .help("Analysiert mehrere Frames mit Apple Vision und berechnet einen statischen Crop pro Clip. Kein Frame-by-frame Tracking.")

@@ -137,6 +137,7 @@ struct RenderSettings: Equatable {
    var stabilizationAnchorX: Double = 0.5
    var stabilizationAnchorY: Double = 0.5
    var stabilizationAnchorSize: Double = 0.35
+    var bestSegmentDetectionEnabled: Bool = false
    var faceNormalizationEnabled: Bool = false
    var targetFaceHeightRatio: Double = 0.28
 }
@@ -168,6 +169,7 @@ struct PersistedRenderSettings: Codable {
    var stabilizationAnchorX: Double
    var stabilizationAnchorY: Double
    var stabilizationAnchorSize: Double
+    var bestSegmentDetectionEnabled: Bool?
    var faceNormalizationEnabled: Bool
    var targetFaceHeightRatio: Double

@@ -198,6 +200,7 @@ struct PersistedRenderSettings: Codable {
        stabilizationAnchorX = settings.stabilizationAnchorX
        stabilizationAnchorY = settings.stabilizationAnchorY
        stabilizationAnchorSize = settings.stabilizationAnchorSize
+        bestSegmentDetectionEnabled = settings.bestSegmentDetectionEnabled
        faceNormalizationEnabled = settings.faceNormalizationEnabled
        targetFaceHeightRatio = settings.targetFaceHeightRatio
    }
@@ -230,6 +233,7 @@ struct PersistedRenderSettings: Codable {
            stabilizationAnchorX: stabilizationAnchorX,
            stabilizationAnchorY: stabilizationAnchorY,
            stabilizationAnchorSize: stabilizationAnchorSize,
+            bestSegmentDetectionEnabled: bestSegmentDetectionEnabled ?? false,
            faceNormalizationEnabled: faceNormalizationEnabled,
            targetFaceHeightRatio: targetFaceHeightRatio
        )
@@ -209,13 +209,18 @@ final class VideoProcessor: ObservableObject {
        let targetDimensions = outputDimensions(for: effectiveSettings, firstVideo: videos[0])
        appendLog("Zielauflösung der Zwischenclips: \(targetDimensions.width)x\(targetDimensions.height)")

-        var renderProject = existingProject ?? makeProject(
+        var renderProject: GrowthLapseProject
+        if let existingProject {
+            renderProject = existingProject
+        } else {
+            renderProject = try await makeProject(
                outputFile: outputFile,
                cacheDirectory: tempDirectory,
                targetDimensions: targetDimensions,
                videos: videos,
                settings: effectiveSettings
            )
+        }

        let normalizedClips = try await normalizeVideos(
            videos,
@@ -1045,18 +1050,21 @@ final class VideoProcessor: ObservableObject {
        targetDimensions: VideoDimensions,
        videos: [VideoFile],
        settings: RenderSettings
-    ) -> GrowthLapseProject {
-        let clips = videos.enumerated().map { index, video in
+    ) async throws -> GrowthLapseProject {
+        var clips: [GrowthLapseProjectClip] = []
+
+        for (index, video) in videos.enumerated() {
+            try Task.checkCancellation()
            let segmentLength = min(settings.segmentLength, video.duration)
-            let rawStart: Double
-            if settings.takeMiddleSegment, video.duration > settings.segmentLength {
-                rawStart = max(0, (video.duration - settings.segmentLength) / 2 + settings.startOffset)
-            } else {
-                rawStart = max(0, settings.startOffset)
-            }
-            let safeStart = min(rawStart, max(0, video.duration - segmentLength))
+            let safeStart = try await initialSegmentStart(
+                for: video,
+                segmentLength: segmentLength,
+                settings: settings,
+                clipNumber: index + 1,
+                totalClips: videos.count
+            )
            let normalizedClipURL = cacheDirectory.appendingPathComponent(String(format: "clip_%04d.mp4", index + 1))
-            return GrowthLapseProjectClip(
+            clips.append(GrowthLapseProjectClip(
                index: index + 1,
                sourcePath: video.url.path,
                displayName: video.displayName,
@@ -1069,7 +1077,7 @@ final class VideoProcessor: ObservableObject {
                normalizedClipPath: normalizedClipURL.path,
                needsRender: true,
                variants: video.variants.isEmpty ? nil : video.variants
-            )
+            ))
        }

        return GrowthLapseProject(
@@ -1081,6 +1089,45 @@ final class VideoProcessor: ObservableObject {
        )
    }

+    private func initialSegmentStart(
+        for video: VideoFile,
+        segmentLength: Double,
+        settings: RenderSettings,
+        clipNumber: Int,
+        totalClips: Int
+    ) async throws -> Double {
+        let fallbackStart: Double
+        if settings.takeMiddleSegment, video.duration > settings.segmentLength {
+            fallbackStart = max(0, (video.duration - settings.segmentLength) / 2 + settings.startOffset)
+        } else {
+            fallbackStart = max(0, settings.startOffset)
+        }
+        let safeFallback = min(fallbackStart, max(0, video.duration - segmentLength))
+
+        guard settings.bestSegmentDetectionEnabled, video.duration > segmentLength else {
+            return safeFallback
+        }
+
+        progressText = "Bestes Segment \(clipNumber) von \(totalClips)"
+        appendLog("Bestes Segment: analysiere Augen/Gesicht in \(video.displayName)")
+        do {
+            let result = try await BestSegmentAnalyzer.detectBestSegment(
+                source: video.url,
+                duration: video.duration,
+                segmentLength: segmentLength
+            )
+            guard let result else {
+                appendLog("Bestes Segment Clip \(clipNumber): nichts Sicheres gefunden, nutze Fallback @ \(formatSeconds(safeFallback))")
+                return safeFallback
+            }
+            appendLog("Bestes Segment Clip \(clipNumber): Start \(formatSeconds(result.start)), gute Frames \(result.goodFrameCount)/\(result.totalFrameCount), Score \(String(format: "%.2f", locale: Locale(identifier: "en_US_POSIX"), result.score))")
+            return result.start
+        } catch {
+            appendLog("Warnung: Bestes Segment Clip \(clipNumber) fehlgeschlagen: \(error.localizedDescription). Nutze Fallback @ \(formatSeconds(safeFallback))")
+            return safeFallback
+        }
+    }
+
    private func reorderedProject(_ project: GrowthLapseProject, sortMode: SortMode) -> GrowthLapseProject {
        var copy = project
        copy.clips = sortedProjectClips(project.clips, sortMode: sortMode).enumerated().map { offset, clip in
@@ -1278,6 +1325,7 @@ final class VideoProcessor: ObservableObject {
            "Übergangslänge: \(settings.transitionLength)s",
            "Audio entfernen: \(settings.removeAudio ? "ja" : "nein")",
            "Segment aus Mitte: \(settings.takeMiddleSegment ? "ja" : "nein")",
+            "Bestes Segment automatisch: \(settings.bestSegmentDetectionEnabled ? "an, Gesicht/Augen sichtbar" : "aus")",
            "Stabilisierung: \(settings.stabilizationEnabled ? settings.stabilizationMethod.rawValue : "aus")",
            "Stabilisierungsstärke: \(settings.stabilizationStrength.rawValue)",
            "Stabilisierung Hintergrund-Anker: \(settings.stabilizationAnchorEnabled ? "an, X \(Int(settings.stabilizationAnchorX * 100))%, Y \(Int(settings.stabilizationAnchorY * 100))%, Bereich \(Int(settings.stabilizationAnchorSize * 100))%" : "aus")",
@@ -1413,6 +1461,165 @@ private struct DetectedFaceSample {
    let imageHeight: Int
 }

+private struct BestSegmentResult {
+    let start: Double
+    let score: Double
+    let goodFrameCount: Int
+    let totalFrameCount: Int
+}
+
+private struct EyeVisibilitySample {
+    let time: Double
+    let score: Double
+    let isGood: Bool
+}
+
+private enum BestSegmentAnalyzer {
+    static func detectBestSegment(
+        source: URL,
+        duration: Double,
+        segmentLength: Double
+    ) async throws -> BestSegmentResult? {
+        try await Task.detached(priority: .userInitiated) {
+            try Task.checkCancellation()
+
+            let sampleInterval = duration <= 60 ? 0.5 : 1.0
+            let samples = try analyzeSamples(
+                source: source,
+                duration: duration,
+                interval: sampleInterval
+            )
+            let goodSamples = samples.filter(\.isGood)
+            guard !samples.isEmpty, !goodSamples.isEmpty else {
+                return nil
+            }
+
+            let maxStart = max(0, duration - segmentLength)
+            var best: BestSegmentResult?
+            var start = 0.0
+
+            while start <= maxStart + 0.0001 {
+                try Task.checkCancellation()
+                let end = start + segmentLength
+                let windowSamples = samples.filter { $0.time >= start && $0.time <= end }
+                guard !windowSamples.isEmpty else {
+                    start += sampleInterval
+                    continue
+                }
+
+                let goodCount = windowSamples.filter(\.isGood).count
+                let averageScore = windowSamples.map(\.score).reduce(0, +) / Double(windowSamples.count)
+                let goodRatio = Double(goodCount) / Double(windowSamples.count)
+                let longestRun = longestGoodRun(in: windowSamples)
+                let windowScore = Double(goodCount) * 1_000
+                    + Double(longestRun) * 180
+                    + goodRatio * 120
+                    + averageScore * 25
+
+                let result = BestSegmentResult(
+                    start: min(max(0, start), maxStart),
+                    score: windowScore,
+                    goodFrameCount: goodCount,
+                    totalFrameCount: windowSamples.count
+                )
+
+                if let currentBest = best {
+                    if result.score > currentBest.score {
+                        best = result
+                    }
+                } else {
+                    best = result
+                }
+
+                start += sampleInterval
+            }
+
+            guard let best, best.goodFrameCount > 0 else {
+                return nil
+            }
+            return best
+        }.value
+    }
+
+    private static func analyzeSamples(source: URL, duration: Double, interval: Double) throws -> [EyeVisibilitySample] {
+        let asset = AVAsset(url: source)
+        let generator = AVAssetImageGenerator(asset: asset)
+        generator.appliesPreferredTrackTransform = true
+        generator.maximumSize = CGSize(width: 960, height: 960)
+        generator.requestedTimeToleranceBefore = CMTime(seconds: 0.12, preferredTimescale: 600)
+        generator.requestedTimeToleranceAfter = CMTime(seconds: 0.12, preferredTimescale: 600)
+
+        var samples: [EyeVisibilitySample] = []
+        var time = 0.0
+        while time <= duration {
+            try Task.checkCancellation()
+            let cmTime = CMTime(seconds: time, preferredTimescale: 600)
+            let cgImage = try generator.copyCGImage(at: cmTime, actualTime: nil)
+            let score = try eyeVisibilityScore(in: cgImage)
+            samples.append(EyeVisibilitySample(time: time, score: score, isGood: score >= 0.62))
+            time += interval
+        }
+        return samples
+    }
+
+    private static func eyeVisibilityScore(in image: CGImage) throws -> Double {
+        let request = VNDetectFaceLandmarksRequest()
+        let handler = VNImageRequestHandler(cgImage: image, options: [:])
+        try handler.perform([request])
+
+        guard let faces = request.results, !faces.isEmpty else {
+            return 0
+        }
+
+        return faces.map(scoreFace).max() ?? 0
+    }
+
+    private static func scoreFace(_ face: VNFaceObservation) -> Double {
+        let faceArea = Double(face.boundingBox.width * face.boundingBox.height)
+        let faceSizeScore = clamp(faceArea / 0.035, min: 0, max: 1)
+        let centerDistance = hypot(Double(face.boundingBox.midX - 0.5), Double(face.boundingBox.midY - 0.5))
+        let centerScore = clamp(1 - centerDistance * 1.4, min: 0, max: 1)
+
+        let leftEyePoints = face.landmarks?.leftEye?.pointCount ?? 0
+        let rightEyePoints = face.landmarks?.rightEye?.pointCount ?? 0
+        let hasBothEyes = leftEyePoints >= 4 && rightEyePoints >= 4
+        let eyePointScore = clamp(Double(min(leftEyePoints, rightEyePoints)) / 8.0, min: 0, max: 1)
+
+        let yawPenalty = min(abs(face.yaw?.doubleValue ?? 0) / 0.55, 1)
+        let rollPenalty = min(abs(face.roll?.doubleValue ?? 0) / 0.45, 1)
+        let poseScore = clamp(1 - yawPenalty * 0.85 - rollPenalty * 0.35, min: 0, max: 1)
+        let qualityScore = face.faceCaptureQuality.map(Double.init) ?? 0.55
+
+        guard hasBothEyes else {
+            return faceSizeScore * 0.20 + centerScore * 0.08 + poseScore * 0.12
+        }
+
+        return eyePointScore * 0.46
+            + poseScore * 0.22
+            + faceSizeScore * 0.14
+            + qualityScore * 0.12
+            + centerScore * 0.06
+    }
+
+    private static func longestGoodRun(in samples: [EyeVisibilitySample]) -> Int {
+        var best = 0
+        var current = 0
+        for sample in samples {
+            if sample.isGood {
+                current += 1
+                best = max(best, current)
+            } else {
+                current = 0
+            }
+        }
+        return best
+    }
+
+    private static func clamp(_ value: Double, min minimum: Double, max maximum: Double) -> Double {
+        Swift.max(minimum, Swift.min(maximum, value))
+    }
+}
+
 private enum FaceCropAnalyzer {
    static func detectCrop(
        source: URL,