From 8f57d1f429f4299abaf08c16ffe482eef73db397 Mon Sep 17 00:00:00 2001 From: Mikei386 <44135113+Mikei386@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:17:49 +0200 Subject: [PATCH] Add automatic best segment detection --- Sources/GrowthLapse/ContentView.swift | 3 + Sources/GrowthLapse/RenderSettings.swift | 4 + Sources/GrowthLapse/VideoProcessor.swift | 243 +++++++++++++++++++++-- 3 files changed, 232 insertions(+), 18 deletions(-) diff --git a/Sources/GrowthLapse/ContentView.swift b/Sources/GrowthLapse/ContentView.swift index 31aa7d9..8977d91 100644 --- a/Sources/GrowthLapse/ContentView.swift +++ b/Sources/GrowthLapse/ContentView.swift @@ -175,6 +175,9 @@ struct ContentView: View { .disabled(processor.state.isRunning) Section("Gesicht") { + Toggle("Bestes Segment automatisch finden", isOn: $settings.bestSegmentDetectionEnabled) + .help("Analysiert das Originalvideo vor dem Schneiden und sucht das Fenster, in dem Gesicht und beide Augen am besten sichtbar sind. Wenn nichts Brauchbares gefunden wird, nutzt GrowthLapse die normale Mitte/Start-Offset-Logik.") + Toggle("Gesichtsgröße normalisieren", isOn: $settings.faceNormalizationEnabled) .help("Analysiert mehrere Frames mit Apple Vision und berechnet einen statischen Crop pro Clip. Kein Frame-by-frame Tracking.") diff --git a/Sources/GrowthLapse/RenderSettings.swift b/Sources/GrowthLapse/RenderSettings.swift index 0a8f43c..cb4faa0 100644 --- a/Sources/GrowthLapse/RenderSettings.swift +++ b/Sources/GrowthLapse/RenderSettings.swift @@ -137,6 +137,7 @@ struct RenderSettings: Equatable { var stabilizationAnchorX: Double = 0.5 var stabilizationAnchorY: Double = 0.5 var stabilizationAnchorSize: Double = 0.35 + var bestSegmentDetectionEnabled: Bool = false var faceNormalizationEnabled: Bool = false var targetFaceHeightRatio: Double = 0.28 } @@ -168,6 +169,7 @@ struct PersistedRenderSettings: Codable { var stabilizationAnchorX: Double var stabilizationAnchorY: Double var stabilizationAnchorSize: Double + var bestSegmentDetectionEnabled: Bool? var faceNormalizationEnabled: Bool var targetFaceHeightRatio: Double @@ -198,6 +200,7 @@ struct PersistedRenderSettings: Codable { stabilizationAnchorX = settings.stabilizationAnchorX stabilizationAnchorY = settings.stabilizationAnchorY stabilizationAnchorSize = settings.stabilizationAnchorSize + bestSegmentDetectionEnabled = settings.bestSegmentDetectionEnabled faceNormalizationEnabled = settings.faceNormalizationEnabled targetFaceHeightRatio = settings.targetFaceHeightRatio } @@ -230,6 +233,7 @@ struct PersistedRenderSettings: Codable { stabilizationAnchorX: stabilizationAnchorX, stabilizationAnchorY: stabilizationAnchorY, stabilizationAnchorSize: stabilizationAnchorSize, + bestSegmentDetectionEnabled: bestSegmentDetectionEnabled ?? false, faceNormalizationEnabled: faceNormalizationEnabled, targetFaceHeightRatio: targetFaceHeightRatio ) diff --git a/Sources/GrowthLapse/VideoProcessor.swift b/Sources/GrowthLapse/VideoProcessor.swift index e477875..df2f411 100644 --- a/Sources/GrowthLapse/VideoProcessor.swift +++ b/Sources/GrowthLapse/VideoProcessor.swift @@ -209,13 +209,18 @@ final class VideoProcessor: ObservableObject { let targetDimensions = outputDimensions(for: effectiveSettings, firstVideo: videos[0]) appendLog("Zielauflösung der Zwischenclips: \(targetDimensions.width)x\(targetDimensions.height)") - var renderProject = existingProject ?? makeProject( - outputFile: outputFile, - cacheDirectory: tempDirectory, - targetDimensions: targetDimensions, - videos: videos, - settings: effectiveSettings - ) + var renderProject: GrowthLapseProject + if let existingProject { + renderProject = existingProject + } else { + renderProject = try await makeProject( + outputFile: outputFile, + cacheDirectory: tempDirectory, + targetDimensions: targetDimensions, + videos: videos, + settings: effectiveSettings + ) + } let normalizedClips = try await normalizeVideos( videos, @@ -1045,18 +1050,21 @@ final class VideoProcessor: ObservableObject { targetDimensions: VideoDimensions, videos: [VideoFile], settings: RenderSettings - ) -> GrowthLapseProject { - let clips = videos.enumerated().map { index, video in + ) async throws -> GrowthLapseProject { + var clips: [GrowthLapseProjectClip] = [] + + for (index, video) in videos.enumerated() { + try Task.checkCancellation() let segmentLength = min(settings.segmentLength, video.duration) - let rawStart: Double - if settings.takeMiddleSegment, video.duration > settings.segmentLength { - rawStart = max(0, (video.duration - settings.segmentLength) / 2 + settings.startOffset) - } else { - rawStart = max(0, settings.startOffset) - } - let safeStart = min(rawStart, max(0, video.duration - segmentLength)) + let safeStart = try await initialSegmentStart( + for: video, + segmentLength: segmentLength, + settings: settings, + clipNumber: index + 1, + totalClips: videos.count + ) let normalizedClipURL = cacheDirectory.appendingPathComponent(String(format: "clip_%04d.mp4", index + 1)) - return GrowthLapseProjectClip( + clips.append(GrowthLapseProjectClip( index: index + 1, sourcePath: video.url.path, displayName: video.displayName, @@ -1069,7 +1077,7 @@ final class VideoProcessor: ObservableObject { normalizedClipPath: normalizedClipURL.path, needsRender: true, variants: video.variants.isEmpty ? nil : video.variants - ) + )) } return GrowthLapseProject( @@ -1081,6 +1089,45 @@ final class VideoProcessor: ObservableObject { ) } + private func initialSegmentStart( + for video: VideoFile, + segmentLength: Double, + settings: RenderSettings, + clipNumber: Int, + totalClips: Int + ) async throws -> Double { + let fallbackStart: Double + if settings.takeMiddleSegment, video.duration > settings.segmentLength { + fallbackStart = max(0, (video.duration - settings.segmentLength) / 2 + settings.startOffset) + } else { + fallbackStart = max(0, settings.startOffset) + } + let safeFallback = min(fallbackStart, max(0, video.duration - segmentLength)) + + guard settings.bestSegmentDetectionEnabled, video.duration > segmentLength else { + return safeFallback + } + + progressText = "Bestes Segment \(clipNumber) von \(totalClips)" + appendLog("Bestes Segment: analysiere Augen/Gesicht in \(video.displayName)") + do { + let result = try await BestSegmentAnalyzer.detectBestSegment( + source: video.url, + duration: video.duration, + segmentLength: segmentLength + ) + guard let result else { + appendLog("Bestes Segment Clip \(clipNumber): nichts Sicheres gefunden, nutze Fallback @ \(formatSeconds(safeFallback))") + return safeFallback + } + appendLog("Bestes Segment Clip \(clipNumber): Start \(formatSeconds(result.start)), gute Frames \(result.goodFrameCount)/\(result.totalFrameCount), Score \(String(format: "%.2f", locale: Locale(identifier: "en_US_POSIX"), result.score))") + return result.start + } catch { + appendLog("Warnung: Bestes Segment Clip \(clipNumber) fehlgeschlagen: \(error.localizedDescription). Nutze Fallback @ \(formatSeconds(safeFallback))") + return safeFallback + } + } + private func reorderedProject(_ project: GrowthLapseProject, sortMode: SortMode) -> GrowthLapseProject { var copy = project copy.clips = sortedProjectClips(project.clips, sortMode: sortMode).enumerated().map { offset, clip in @@ -1278,6 +1325,7 @@ final class VideoProcessor: ObservableObject { "Übergangslänge: \(settings.transitionLength)s", "Audio entfernen: \(settings.removeAudio ? "ja" : "nein")", "Segment aus Mitte: \(settings.takeMiddleSegment ? "ja" : "nein")", + "Bestes Segment automatisch: \(settings.bestSegmentDetectionEnabled ? "an, Gesicht/Augen sichtbar" : "aus")", "Stabilisierung: \(settings.stabilizationEnabled ? settings.stabilizationMethod.rawValue : "aus")", "Stabilisierungsstärke: \(settings.stabilizationStrength.rawValue)", "Stabilisierung Hintergrund-Anker: \(settings.stabilizationAnchorEnabled ? "an, X \(Int(settings.stabilizationAnchorX * 100))%, Y \(Int(settings.stabilizationAnchorY * 100))%, Bereich \(Int(settings.stabilizationAnchorSize * 100))%" : "aus")", @@ -1413,6 +1461,165 @@ private struct DetectedFaceSample { let imageHeight: Int } +private struct BestSegmentResult { + let start: Double + let score: Double + let goodFrameCount: Int + let totalFrameCount: Int +} + +private struct EyeVisibilitySample { + let time: Double + let score: Double + let isGood: Bool +} + +private enum BestSegmentAnalyzer { + static func detectBestSegment( + source: URL, + duration: Double, + segmentLength: Double + ) async throws -> BestSegmentResult? { + try await Task.detached(priority: .userInitiated) { + try Task.checkCancellation() + + let sampleInterval = duration <= 60 ? 0.5 : 1.0 + let samples = try analyzeSamples( + source: source, + duration: duration, + interval: sampleInterval + ) + let goodSamples = samples.filter(\.isGood) + guard !samples.isEmpty, !goodSamples.isEmpty else { + return nil + } + + let maxStart = max(0, duration - segmentLength) + var best: BestSegmentResult? + var start = 0.0 + + while start <= maxStart + 0.0001 { + try Task.checkCancellation() + let end = start + segmentLength + let windowSamples = samples.filter { $0.time >= start && $0.time <= end } + guard !windowSamples.isEmpty else { + start += sampleInterval + continue + } + + let goodCount = windowSamples.filter(\.isGood).count + let averageScore = windowSamples.map(\.score).reduce(0, +) / Double(windowSamples.count) + let goodRatio = Double(goodCount) / Double(windowSamples.count) + let longestRun = longestGoodRun(in: windowSamples) + let windowScore = Double(goodCount) * 1_000 + + Double(longestRun) * 180 + + goodRatio * 120 + + averageScore * 25 + + let result = BestSegmentResult( + start: min(max(0, start), maxStart), + score: windowScore, + goodFrameCount: goodCount, + totalFrameCount: windowSamples.count + ) + + if let currentBest = best { + if result.score > currentBest.score { + best = result + } + } else { + best = result + } + + start += sampleInterval + } + + guard let best, best.goodFrameCount > 0 else { + return nil + } + return best + }.value + } + + private static func analyzeSamples(source: URL, duration: Double, interval: Double) throws -> [EyeVisibilitySample] { + let asset = AVAsset(url: source) + let generator = AVAssetImageGenerator(asset: asset) + generator.appliesPreferredTrackTransform = true + generator.maximumSize = CGSize(width: 960, height: 960) + generator.requestedTimeToleranceBefore = CMTime(seconds: 0.12, preferredTimescale: 600) + generator.requestedTimeToleranceAfter = CMTime(seconds: 0.12, preferredTimescale: 600) + + var samples: [EyeVisibilitySample] = [] + var time = 0.0 + while time <= duration { + try Task.checkCancellation() + let cmTime = CMTime(seconds: time, preferredTimescale: 600) + let cgImage = try generator.copyCGImage(at: cmTime, actualTime: nil) + let score = try eyeVisibilityScore(in: cgImage) + samples.append(EyeVisibilitySample(time: time, score: score, isGood: score >= 0.62)) + time += interval + } + return samples + } + + private static func eyeVisibilityScore(in image: CGImage) throws -> Double { + let request = VNDetectFaceLandmarksRequest() + let handler = VNImageRequestHandler(cgImage: image, options: [:]) + try handler.perform([request]) + + guard let faces = request.results, !faces.isEmpty else { + return 0 + } + + return faces.map(scoreFace).max() ?? 0 + } + + private static func scoreFace(_ face: VNFaceObservation) -> Double { + let faceArea = Double(face.boundingBox.width * face.boundingBox.height) + let faceSizeScore = clamp(faceArea / 0.035, min: 0, max: 1) + let centerDistance = hypot(Double(face.boundingBox.midX - 0.5), Double(face.boundingBox.midY - 0.5)) + let centerScore = clamp(1 - centerDistance * 1.4, min: 0, max: 1) + + let leftEyePoints = face.landmarks?.leftEye?.pointCount ?? 0 + let rightEyePoints = face.landmarks?.rightEye?.pointCount ?? 0 + let hasBothEyes = leftEyePoints >= 4 && rightEyePoints >= 4 + let eyePointScore = clamp(Double(min(leftEyePoints, rightEyePoints)) / 8.0, min: 0, max: 1) + + let yawPenalty = min(abs(face.yaw?.doubleValue ?? 0) / 0.55, 1) + let rollPenalty = min(abs(face.roll?.doubleValue ?? 0) / 0.45, 1) + let poseScore = clamp(1 - yawPenalty * 0.85 - rollPenalty * 0.35, min: 0, max: 1) + let qualityScore = face.faceCaptureQuality.map(Double.init) ?? 0.55 + + guard hasBothEyes else { + return faceSizeScore * 0.20 + centerScore * 0.08 + poseScore * 0.12 + } + + return eyePointScore * 0.46 + + poseScore * 0.22 + + faceSizeScore * 0.14 + + qualityScore * 0.12 + + centerScore * 0.06 + } + + private static func longestGoodRun(in samples: [EyeVisibilitySample]) -> Int { + var best = 0 + var current = 0 + for sample in samples { + if sample.isGood { + current += 1 + best = max(best, current) + } else { + current = 0 + } + } + return best + } + + private static func clamp(_ value: Double, min minimum: Double, max maximum: Double) -> Double { + Swift.max(minimum, Swift.min(maximum, value)) + } +} + private enum FaceCropAnalyzer { static func detectCrop( source: URL,