Add photo morphing render mode
This commit is contained in:
@@ -13,11 +13,20 @@ struct ContentView: View {
|
|||||||
HSplitView {
|
HSplitView {
|
||||||
Form {
|
Form {
|
||||||
Section("Dateien") {
|
Section("Dateien") {
|
||||||
|
Picker("Modus", selection: $settings.renderMode) {
|
||||||
|
ForEach(RenderMode.allCases) { mode in
|
||||||
|
Text(mode.rawValue).tag(mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.help("Video-Zeitraffer verarbeitet MOV/MP4-Clips. Foto-Morphing verarbeitet Bilder aus dem Input-Ordner und erzeugt Landmark-basierte Morph-Übergänge.")
|
||||||
|
|
||||||
pickerRow(
|
pickerRow(
|
||||||
title: "Input-Ordner",
|
title: "Input-Ordner",
|
||||||
value: settings.inputFolder?.path(percentEncoded: false) ?? "Nicht gewählt",
|
value: settings.inputFolder?.path(percentEncoded: false) ?? "Nicht gewählt",
|
||||||
buttonTitle: "Auswählen",
|
buttonTitle: "Auswählen",
|
||||||
help: "Ordner mit den chronologisch sortierten Quellvideos. Die gewählte Output-Datei wird automatisch aus der Input-Liste ausgeschlossen."
|
help: settings.renderMode == .photoMorphing
|
||||||
|
? "Ordner mit chronologisch sortierten Bildern für Foto-Morphing."
|
||||||
|
: "Ordner mit den chronologisch sortierten Quellvideos. Die gewählte Output-Datei wird automatisch aus der Input-Liste ausgeschlossen."
|
||||||
) {
|
) {
|
||||||
selectInputFolder()
|
selectInputFolder()
|
||||||
}
|
}
|
||||||
@@ -65,8 +74,10 @@ struct ContentView: View {
|
|||||||
.disabled(processor.state.isRunning)
|
.disabled(processor.state.isRunning)
|
||||||
|
|
||||||
Section("Parameter") {
|
Section("Parameter") {
|
||||||
doubleStepper("Ausschnittlänge", value: $settings.segmentLength, range: 1...120, step: 1, suffix: "s", help: "So viele Sekunden werden aus jedem Originalvideo entnommen, bevor Stabilisierung und Beschleunigung passieren.")
|
if settings.renderMode == .videoGrowthLapse {
|
||||||
doubleStepper("Ziellänge pro Clip", value: $settings.targetClipLength, range: 0.5...30, step: 0.5, suffix: "s", help: "Länge jedes fertigen Zwischenclips nach der Beschleunigung. Beispiel: 10 Sekunden Segment auf 2 Sekunden ergibt 5x Speed.")
|
doubleStepper("Ausschnittlänge", value: $settings.segmentLength, range: 1...120, step: 1, suffix: "s", help: "So viele Sekunden werden aus jedem Originalvideo entnommen, bevor Stabilisierung und Beschleunigung passieren.")
|
||||||
|
}
|
||||||
|
doubleStepper("Ziellänge pro Clip", value: $settings.targetClipLength, range: 0.5...30, step: 0.5, suffix: "s", help: settings.renderMode == .photoMorphing ? "Dauer des Morph-Übergangs von einem Bild zum nächsten." : "Länge jedes fertigen Zwischenclips nach der Beschleunigung. Beispiel: 10 Sekunden Segment auf 2 Sekunden ergibt 5x Speed.")
|
||||||
doubleStepper("Übergangslänge", value: $settings.transitionLength, range: 0...5, step: 0.1, suffix: "s", help: "Dauer des Crossfade zwischen zwei Clips. 0 verbindet hart per concat.")
|
doubleStepper("Übergangslänge", value: $settings.transitionLength, range: 0...5, step: 0.1, suffix: "s", help: "Dauer des Crossfade zwischen zwei Clips. 0 verbindet hart per concat.")
|
||||||
Stepper(value: $settings.fps, in: 1...120) {
|
Stepper(value: $settings.fps, in: 1...120) {
|
||||||
HStack {
|
HStack {
|
||||||
@@ -86,7 +97,9 @@ struct ContentView: View {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
.help("Qualitätswert für libx264. Niedriger ist bessere Qualität/größere Datei. Wird bei VideoToolbox nicht verwendet.")
|
.help("Qualitätswert für libx264. Niedriger ist bessere Qualität/größere Datei. Wird bei VideoToolbox nicht verwendet.")
|
||||||
doubleStepper("Start-Offset", value: $settings.startOffset, range: -60...60, step: 0.5, suffix: "s", help: "Verschiebt den gewählten Ausschnitt relativ zur Mitte. Positive Werte nehmen späteres Material, negative früheres.")
|
if settings.renderMode == .videoGrowthLapse {
|
||||||
|
doubleStepper("Start-Offset", value: $settings.startOffset, range: -60...60, step: 0.5, suffix: "s", help: "Verschiebt den gewählten Ausschnitt relativ zur Mitte. Positive Werte nehmen späteres Material, negative früheres.")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
.disabled(processor.state.isRunning)
|
.disabled(processor.state.isRunning)
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,12 @@
|
|||||||
import Foundation
|
import Foundation
|
||||||
|
|
||||||
|
enum RenderMode: String, CaseIterable, Identifiable, Codable {
|
||||||
|
case videoGrowthLapse = "Video-Zeitraffer"
|
||||||
|
case photoMorphing = "Foto-Morphing"
|
||||||
|
|
||||||
|
var id: String { rawValue }
|
||||||
|
}
|
||||||
|
|
||||||
enum OutputFormat: String, CaseIterable, Identifiable, Codable {
|
enum OutputFormat: String, CaseIterable, Identifiable, Codable {
|
||||||
case landscape1920x1080 = "Landscape 1920x1080 SDR"
|
case landscape1920x1080 = "Landscape 1920x1080 SDR"
|
||||||
case portrait1080x1920 = "Portrait 1080x1920"
|
case portrait1080x1920 = "Portrait 1080x1920"
|
||||||
@@ -115,6 +122,7 @@ struct RenderSettings: Equatable {
|
|||||||
var outputFile: URL?
|
var outputFile: URL?
|
||||||
var ffmpegExecutable: URL?
|
var ffmpegExecutable: URL?
|
||||||
var ffprobeExecutable: URL?
|
var ffprobeExecutable: URL?
|
||||||
|
var renderMode: RenderMode = .videoGrowthLapse
|
||||||
var segmentLength: Double = 20
|
var segmentLength: Double = 20
|
||||||
var targetClipLength: Double = 3
|
var targetClipLength: Double = 3
|
||||||
var transitionLength: Double = 0.5
|
var transitionLength: Double = 0.5
|
||||||
@@ -147,6 +155,7 @@ struct PersistedRenderSettings: Codable {
|
|||||||
var outputFilePath: String?
|
var outputFilePath: String?
|
||||||
var ffmpegExecutablePath: String?
|
var ffmpegExecutablePath: String?
|
||||||
var ffprobeExecutablePath: String?
|
var ffprobeExecutablePath: String?
|
||||||
|
var renderMode: RenderMode?
|
||||||
var segmentLength: Double
|
var segmentLength: Double
|
||||||
var targetClipLength: Double
|
var targetClipLength: Double
|
||||||
var transitionLength: Double
|
var transitionLength: Double
|
||||||
@@ -178,6 +187,7 @@ struct PersistedRenderSettings: Codable {
|
|||||||
outputFilePath = settings.outputFile?.path
|
outputFilePath = settings.outputFile?.path
|
||||||
ffmpegExecutablePath = settings.ffmpegExecutable?.path
|
ffmpegExecutablePath = settings.ffmpegExecutable?.path
|
||||||
ffprobeExecutablePath = settings.ffprobeExecutable?.path
|
ffprobeExecutablePath = settings.ffprobeExecutable?.path
|
||||||
|
renderMode = settings.renderMode
|
||||||
segmentLength = settings.segmentLength
|
segmentLength = settings.segmentLength
|
||||||
targetClipLength = settings.targetClipLength
|
targetClipLength = settings.targetClipLength
|
||||||
transitionLength = settings.transitionLength
|
transitionLength = settings.transitionLength
|
||||||
@@ -211,6 +221,7 @@ struct PersistedRenderSettings: Codable {
|
|||||||
outputFile: outputFilePath.map { URL(fileURLWithPath: $0) },
|
outputFile: outputFilePath.map { URL(fileURLWithPath: $0) },
|
||||||
ffmpegExecutable: ffmpegExecutablePath.map { URL(fileURLWithPath: $0) },
|
ffmpegExecutable: ffmpegExecutablePath.map { URL(fileURLWithPath: $0) },
|
||||||
ffprobeExecutable: ffprobeExecutablePath.map { URL(fileURLWithPath: $0) },
|
ffprobeExecutable: ffprobeExecutablePath.map { URL(fileURLWithPath: $0) },
|
||||||
|
renderMode: renderMode ?? .videoGrowthLapse,
|
||||||
segmentLength: segmentLength,
|
segmentLength: segmentLength,
|
||||||
targetClipLength: targetClipLength,
|
targetClipLength: targetClipLength,
|
||||||
transitionLength: transitionLength,
|
transitionLength: transitionLength,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import AppKit
|
import AppKit
|
||||||
import AVFoundation
|
import AVFoundation
|
||||||
import Foundation
|
import Foundation
|
||||||
|
import ImageIO
|
||||||
import Vision
|
import Vision
|
||||||
|
|
||||||
private struct RenderOutcome {
|
private struct RenderOutcome {
|
||||||
@@ -39,7 +40,7 @@ final class VideoProcessor: ObservableObject {
|
|||||||
guard !Task.isCancelled else {
|
guard !Task.isCancelled else {
|
||||||
throw VideoProcessingError.cancelled
|
throw VideoProcessingError.cancelled
|
||||||
}
|
}
|
||||||
project = outcome.project
|
project = settingsCopy.renderMode == .photoMorphing ? nil : outcome.project
|
||||||
state = .finished(outcome.outputURL)
|
state = .finished(outcome.outputURL)
|
||||||
progress = 1
|
progress = 1
|
||||||
progressText = "Fertig"
|
progressText = "Fertig"
|
||||||
@@ -169,6 +170,10 @@ final class VideoProcessor: ObservableObject {
|
|||||||
appendLog("ffmpeg: \(tools.ffmpeg.path)")
|
appendLog("ffmpeg: \(tools.ffmpeg.path)")
|
||||||
appendLog("ffprobe: \(tools.ffprobe.path)")
|
appendLog("ffprobe: \(tools.ffprobe.path)")
|
||||||
|
|
||||||
|
if effectiveSettings.renderMode == .photoMorphing {
|
||||||
|
return try await renderPhotoMorph(settings: effectiveSettings, ffmpeg: tools.ffmpeg)
|
||||||
|
}
|
||||||
|
|
||||||
if effectiveSettings.stabilizationEnabled {
|
if effectiveSettings.stabilizationEnabled {
|
||||||
let filters = try await service.availableFilters(ffmpeg: tools.ffmpeg)
|
let filters = try await service.availableFilters(ffmpeg: tools.ffmpeg)
|
||||||
effectiveSettings.stabilizationMethod = resolveStabilizationMethod(
|
effectiveSettings.stabilizationMethod = resolveStabilizationMethod(
|
||||||
@@ -264,6 +269,145 @@ final class VideoProcessor: ObservableObject {
|
|||||||
return RenderOutcome(outputURL: outputFile, project: renderProject)
|
return RenderOutcome(outputURL: outputFile, project: renderProject)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private func renderPhotoMorph(settings: RenderSettings, ffmpeg: URL) async throws -> RenderOutcome {
|
||||||
|
let inputFolder = settings.inputFolder!
|
||||||
|
let outputFile = settings.outputFile!
|
||||||
|
let tempDirectory = try createWorkingDirectory(
|
||||||
|
keepDebugFiles: settings.keepIntermediateClips,
|
||||||
|
keepReviewCache: false,
|
||||||
|
outputFile: outputFile,
|
||||||
|
resetCache: true
|
||||||
|
)
|
||||||
|
temporaryDirectory = tempDirectory
|
||||||
|
appendLog("Foto-Morphing: Temporärer Ordner \(tempDirectory.path)")
|
||||||
|
|
||||||
|
let images = try discoverImages(in: inputFolder, sortMode: settings.sortMode)
|
||||||
|
guard images.count >= 2 else {
|
||||||
|
throw VideoProcessingError.invalidSettings("Für Foto-Morphing braucht GrowthLapse mindestens zwei Bilder im Input-Ordner.")
|
||||||
|
}
|
||||||
|
appendLog("Foto-Morphing: \(images.count) Bild(er) gefunden.")
|
||||||
|
|
||||||
|
let targetDimensions = outputDimensionsForStillMode(settings)
|
||||||
|
let framesDirectory = tempDirectory.appendingPathComponent("morph_frames", isDirectory: true)
|
||||||
|
try FileManager.default.createDirectory(at: framesDirectory, withIntermediateDirectories: true)
|
||||||
|
|
||||||
|
progressText = "Analysiere Bilder"
|
||||||
|
let morphImages = try await PhotoMorphRenderer.prepareImages(
|
||||||
|
urls: images,
|
||||||
|
targetDimensions: targetDimensions,
|
||||||
|
targetFaceHeightRatio: settings.targetFaceHeightRatio
|
||||||
|
) { [weak self] index, total, name, usedLandmarks in
|
||||||
|
Task { @MainActor in
|
||||||
|
self?.progressText = "Bildanalyse \(index) von \(total)"
|
||||||
|
self?.appendLog("Foto-Morphing Analyse \(index)/\(total): \(name)\(usedLandmarks ? "" : " (Fallback ohne Gesicht)")")
|
||||||
|
self?.progress = Double(index) / Double(total) * 0.25
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
progressText = "Erzeuge Morph-Frames"
|
||||||
|
let frameCount = try await PhotoMorphRenderer.renderFrames(
|
||||||
|
images: morphImages,
|
||||||
|
outputDirectory: framesDirectory,
|
||||||
|
fps: settings.fps,
|
||||||
|
secondsPerTransition: settings.targetClipLength
|
||||||
|
) { [weak self] frameIndex, totalFrames in
|
||||||
|
Task { @MainActor in
|
||||||
|
self?.progressText = "Morph-Frame \(frameIndex) von \(totalFrames)"
|
||||||
|
self?.progress = 0.25 + Double(frameIndex) / Double(max(1, totalFrames)) * 0.55
|
||||||
|
}
|
||||||
|
}
|
||||||
|
appendLog("Foto-Morphing: \(frameCount) Frame(s) erzeugt.")
|
||||||
|
|
||||||
|
progressText = "Kodieren"
|
||||||
|
try await encodeImageSequence(
|
||||||
|
framesDirectory: framesDirectory,
|
||||||
|
outputFile: outputFile,
|
||||||
|
settings: settings,
|
||||||
|
ffmpeg: ffmpeg
|
||||||
|
)
|
||||||
|
|
||||||
|
progress = 0.95
|
||||||
|
let reportURL = writePhotoMorphReport(
|
||||||
|
settings: settings,
|
||||||
|
outputFile: outputFile,
|
||||||
|
targetDimensions: targetDimensions,
|
||||||
|
images: images,
|
||||||
|
frameCount: frameCount
|
||||||
|
)
|
||||||
|
appendLog("Render-Protokoll: \(reportURL.path)")
|
||||||
|
|
||||||
|
if !settings.keepIntermediateClips {
|
||||||
|
try? FileManager.default.removeItem(at: tempDirectory)
|
||||||
|
}
|
||||||
|
|
||||||
|
return RenderOutcome(outputURL: outputFile, project: GrowthLapseProject(
|
||||||
|
outputFilePath: outputFile.path,
|
||||||
|
cacheDirectoryPath: tempDirectory.path,
|
||||||
|
targetWidth: targetDimensions.width,
|
||||||
|
targetHeight: targetDimensions.height,
|
||||||
|
clips: []
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
private func discoverImages(in folder: URL, sortMode: SortMode) throws -> [URL] {
|
||||||
|
let allowed = Set(["jpg", "jpeg", "png", "heic", "heif", "tif", "tiff"])
|
||||||
|
var urls = try FileManager.default.contentsOfDirectory(
|
||||||
|
at: folder,
|
||||||
|
includingPropertiesForKeys: [.contentModificationDateKey],
|
||||||
|
options: [.skipsHiddenFiles]
|
||||||
|
)
|
||||||
|
.filter { allowed.contains($0.pathExtension.lowercased()) }
|
||||||
|
|
||||||
|
switch sortMode {
|
||||||
|
case .filenameAscending, .ageWeeksAscending:
|
||||||
|
urls.sort {
|
||||||
|
$0.lastPathComponent.localizedStandardCompare($1.lastPathComponent) == .orderedAscending
|
||||||
|
}
|
||||||
|
case .modificationDateAscending:
|
||||||
|
urls.sort {
|
||||||
|
let left = (try? $0.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate) ?? .distantPast
|
||||||
|
let right = (try? $1.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate) ?? .distantPast
|
||||||
|
return left < right
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
guard !urls.isEmpty else {
|
||||||
|
throw VideoProcessingError.invalidSettings("Im Input-Ordner wurden keine Bilder gefunden. Unterstützt: JPG, PNG, HEIC, TIFF.")
|
||||||
|
}
|
||||||
|
return urls
|
||||||
|
}
|
||||||
|
|
||||||
|
private func outputDimensionsForStillMode(_ settings: RenderSettings) -> VideoDimensions {
|
||||||
|
switch settings.outputFormat {
|
||||||
|
case .landscape1920x1080:
|
||||||
|
return VideoDimensions(width: 1920, height: 1080)
|
||||||
|
case .portrait1080x1920:
|
||||||
|
return VideoDimensions(width: 1080, height: 1920)
|
||||||
|
case .keepOriginal:
|
||||||
|
return VideoDimensions(width: 1920, height: 1080)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func encodeImageSequence(framesDirectory: URL, outputFile: URL, settings: RenderSettings, ffmpeg: URL) async throws {
|
||||||
|
var arguments = [
|
||||||
|
"-y",
|
||||||
|
"-nostdin",
|
||||||
|
"-framerate", "\(settings.fps)",
|
||||||
|
"-i", framesDirectory.appendingPathComponent("frame_%06d.png").path
|
||||||
|
]
|
||||||
|
arguments.append(contentsOf: videoEncoderArguments(settings))
|
||||||
|
arguments.append(contentsOf: [
|
||||||
|
"-pix_fmt", "yuv420p",
|
||||||
|
"-movflags", "+faststart",
|
||||||
|
outputFile.path
|
||||||
|
])
|
||||||
|
|
||||||
|
let result = try await runFFmpeg(ffmpeg, arguments: arguments)
|
||||||
|
guard result.exitCode == 0 else {
|
||||||
|
throw VideoProcessingError.ffmpegFailed("Foto-Morphing kodieren", result.output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private func resolveStabilizationMethod(requested: StabilizationMethod, availability: FFmpegFilterAvailability) -> StabilizationMethod {
|
private func resolveStabilizationMethod(requested: StabilizationMethod, availability: FFmpegFilterAvailability) -> StabilizationMethod {
|
||||||
switch requested {
|
switch requested {
|
||||||
case .vidstab:
|
case .vidstab:
|
||||||
@@ -1347,6 +1491,49 @@ final class VideoProcessor: ObservableObject {
|
|||||||
return reportURL
|
return reportURL
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private func writePhotoMorphReport(
|
||||||
|
settings: RenderSettings,
|
||||||
|
outputFile: URL,
|
||||||
|
targetDimensions: VideoDimensions,
|
||||||
|
images: [URL],
|
||||||
|
frameCount: Int
|
||||||
|
) -> URL {
|
||||||
|
let reportURL = outputFile
|
||||||
|
.deletingPathExtension()
|
||||||
|
.appendingPathExtension("growthlapse-report.txt")
|
||||||
|
let imageLines = images.enumerated().map { index, url in
|
||||||
|
String(format: "%03d. %@", index + 1, url.lastPathComponent)
|
||||||
|
}
|
||||||
|
|
||||||
|
let summary = [
|
||||||
|
"GrowthLapse Foto-Morphing Protokoll",
|
||||||
|
"Datum: \(Date())",
|
||||||
|
"Input-Ordner: \(settings.inputFolder?.path ?? "-")",
|
||||||
|
"Output: \(outputFile.path)",
|
||||||
|
"Modus: \(settings.renderMode.rawValue)",
|
||||||
|
"Input-Bilder: \(images.count)",
|
||||||
|
"Frames: \(frameCount)",
|
||||||
|
"Zielauflösung: \(targetDimensions.width)x\(targetDimensions.height)",
|
||||||
|
"FPS: \(settings.fps)",
|
||||||
|
"Morph-Dauer pro Übergang: \(settings.targetClipLength)s",
|
||||||
|
"Ziel-Gesichtshöhe: \(Int(settings.targetFaceHeightRatio * 100))%",
|
||||||
|
"",
|
||||||
|
"Bild-Reihenfolge:",
|
||||||
|
imageLines.joined(separator: "\n"),
|
||||||
|
"",
|
||||||
|
"Log:",
|
||||||
|
logs.joined(separator: "\n")
|
||||||
|
].joined(separator: "\n")
|
||||||
|
|
||||||
|
do {
|
||||||
|
try summary.write(to: reportURL, atomically: true, encoding: .utf8)
|
||||||
|
} catch {
|
||||||
|
appendLog("Warnung: Foto-Morphing-Protokoll konnte nicht geschrieben werden: \(error.localizedDescription)")
|
||||||
|
}
|
||||||
|
|
||||||
|
return reportURL
|
||||||
|
}
|
||||||
|
|
||||||
private func decimal(_ value: Double) -> String {
|
private func decimal(_ value: Double) -> String {
|
||||||
String(format: "%.6f", locale: Locale(identifier: "en_US_POSIX"), value)
|
String(format: "%.6f", locale: Locale(identifier: "en_US_POSIX"), value)
|
||||||
}
|
}
|
||||||
@@ -1461,6 +1648,373 @@ private struct DetectedFaceSample {
|
|||||||
let imageHeight: Int
|
let imageHeight: Int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private struct PreparedMorphImage {
|
||||||
|
let url: URL
|
||||||
|
let image: CGImage
|
||||||
|
let points: [CGPoint]
|
||||||
|
let usedLandmarks: Bool
|
||||||
|
}
|
||||||
|
|
||||||
|
private struct MorphTriangle: Hashable {
|
||||||
|
let a: Int
|
||||||
|
let b: Int
|
||||||
|
let c: Int
|
||||||
|
}
|
||||||
|
|
||||||
|
private enum PhotoMorphRenderer {
|
||||||
|
static func prepareImages(
|
||||||
|
urls: [URL],
|
||||||
|
targetDimensions: VideoDimensions,
|
||||||
|
targetFaceHeightRatio: Double,
|
||||||
|
progress: @escaping (Int, Int, String, Bool) -> Void
|
||||||
|
) async throws -> [PreparedMorphImage] {
|
||||||
|
try await Task.detached(priority: .userInitiated) {
|
||||||
|
var result: [PreparedMorphImage] = []
|
||||||
|
for (index, url) in urls.enumerated() {
|
||||||
|
try Task.checkCancellation()
|
||||||
|
let source = try loadImage(url)
|
||||||
|
let landmarks = try detectLandmarks(in: source)
|
||||||
|
let normalized = normalizeImage(
|
||||||
|
source,
|
||||||
|
landmarks: landmarks,
|
||||||
|
targetDimensions: targetDimensions,
|
||||||
|
targetFaceHeightRatio: targetFaceHeightRatio
|
||||||
|
)
|
||||||
|
result.append(normalized)
|
||||||
|
progress(index + 1, urls.count, url.lastPathComponent, normalized.usedLandmarks)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}.value
|
||||||
|
}
|
||||||
|
|
||||||
|
static func renderFrames(
|
||||||
|
images: [PreparedMorphImage],
|
||||||
|
outputDirectory: URL,
|
||||||
|
fps: Int,
|
||||||
|
secondsPerTransition: Double,
|
||||||
|
progress: @escaping (Int, Int) -> Void
|
||||||
|
) async throws -> Int {
|
||||||
|
try await Task.detached(priority: .userInitiated) {
|
||||||
|
guard images.count >= 2 else {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
let framesPerTransition = max(2, Int((secondsPerTransition * Double(fps)).rounded()))
|
||||||
|
let totalFrames = (images.count - 1) * framesPerTransition + 1
|
||||||
|
var written = 0
|
||||||
|
|
||||||
|
for pairIndex in 0..<(images.count - 1) {
|
||||||
|
try Task.checkCancellation()
|
||||||
|
let left = images[pairIndex]
|
||||||
|
let right = images[pairIndex + 1]
|
||||||
|
let triangles = triangulate(left.points)
|
||||||
|
|
||||||
|
for frameInPair in 0..<framesPerTransition {
|
||||||
|
if pairIndex > 0, frameInPair == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
try Task.checkCancellation()
|
||||||
|
let t = Double(frameInPair) / Double(framesPerTransition)
|
||||||
|
let frame = try renderMorphFrame(left: left, right: right, triangles: triangles, t: t)
|
||||||
|
written += 1
|
||||||
|
try writePNG(frame, to: outputDirectory.appendingPathComponent(String(format: "frame_%06d.png", written)))
|
||||||
|
progress(written, totalFrames)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let last = images[images.count - 1].image
|
||||||
|
written += 1
|
||||||
|
try writePNG(last, to: outputDirectory.appendingPathComponent(String(format: "frame_%06d.png", written)))
|
||||||
|
progress(written, totalFrames)
|
||||||
|
return written
|
||||||
|
}.value
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func loadImage(_ url: URL) throws -> CGImage {
|
||||||
|
guard let source = CGImageSourceCreateWithURL(url as CFURL, nil),
|
||||||
|
let image = CGImageSourceCreateImageAtIndex(source, 0, [
|
||||||
|
kCGImageSourceShouldCache: true
|
||||||
|
] as CFDictionary) else {
|
||||||
|
throw VideoProcessingError.invalidSettings("Bild konnte nicht geladen werden: \(url.lastPathComponent)")
|
||||||
|
}
|
||||||
|
return image
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func detectLandmarks(in image: CGImage) throws -> [CGPoint]? {
|
||||||
|
let request = VNDetectFaceLandmarksRequest()
|
||||||
|
let handler = VNImageRequestHandler(cgImage: image, options: [:])
|
||||||
|
try handler.perform([request])
|
||||||
|
guard let face = request.results?.max(by: { faceScore($0) < faceScore($1) }),
|
||||||
|
let landmarks = face.landmarks else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var points: [CGPoint] = []
|
||||||
|
let regions: [VNFaceLandmarkRegion2D?] = [
|
||||||
|
landmarks.faceContour,
|
||||||
|
landmarks.leftEye,
|
||||||
|
landmarks.rightEye,
|
||||||
|
landmarks.leftEyebrow,
|
||||||
|
landmarks.rightEyebrow,
|
||||||
|
landmarks.nose,
|
||||||
|
landmarks.noseCrest,
|
||||||
|
landmarks.outerLips,
|
||||||
|
landmarks.innerLips,
|
||||||
|
landmarks.medianLine
|
||||||
|
]
|
||||||
|
|
||||||
|
for region in regions.compactMap({ $0 }) {
|
||||||
|
for index in 0..<region.pointCount {
|
||||||
|
let point = region.normalizedPoints[index]
|
||||||
|
points.append(pointInImage(point, face: face, image: image))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
guard points.count >= 8 else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return points
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func normalizeImage(
|
||||||
|
_ source: CGImage,
|
||||||
|
landmarks: [CGPoint]?,
|
||||||
|
targetDimensions: VideoDimensions,
|
||||||
|
targetFaceHeightRatio: Double
|
||||||
|
) -> PreparedMorphImage {
|
||||||
|
let targetSize = CGSize(width: targetDimensions.width, height: targetDimensions.height)
|
||||||
|
let sourceSize = CGSize(width: source.width, height: source.height)
|
||||||
|
|
||||||
|
let transform: CGAffineTransform
|
||||||
|
if let landmarks, !landmarks.isEmpty {
|
||||||
|
let bounds = landmarks.reduce(CGRect.null) { $0.union(CGRect(origin: $1, size: .zero)) }
|
||||||
|
let desiredFaceHeight = targetSize.height * targetFaceHeightRatio
|
||||||
|
let scale = desiredFaceHeight / max(1, bounds.height)
|
||||||
|
let desiredCenter = CGPoint(x: targetSize.width * 0.5, y: targetSize.height * 0.43)
|
||||||
|
transform = CGAffineTransform(translationX: desiredCenter.x, y: desiredCenter.y)
|
||||||
|
.scaledBy(x: scale, y: scale)
|
||||||
|
.translatedBy(x: -bounds.midX, y: -bounds.midY)
|
||||||
|
} else {
|
||||||
|
let scale = min(targetSize.width / sourceSize.width, targetSize.height / sourceSize.height)
|
||||||
|
let x = (targetSize.width - sourceSize.width * scale) / 2
|
||||||
|
let y = (targetSize.height - sourceSize.height * scale) / 2
|
||||||
|
transform = CGAffineTransform(translationX: x, y: y).scaledBy(x: scale, y: scale)
|
||||||
|
}
|
||||||
|
|
||||||
|
let image = drawNormalized(source, targetSize: targetSize, transform: transform)
|
||||||
|
let transformedLandmarks = landmarks?.map { $0.applying(transform) }
|
||||||
|
let points = morphPoints(landmarks: transformedLandmarks, targetSize: targetSize)
|
||||||
|
return PreparedMorphImage(url: URL(fileURLWithPath: ""), image: image, points: points, usedLandmarks: transformedLandmarks != nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func drawNormalized(_ source: CGImage, targetSize: CGSize, transform: CGAffineTransform) -> CGImage {
|
||||||
|
let context = makeContext(width: Int(targetSize.width), height: Int(targetSize.height))
|
||||||
|
context.setFillColor(NSColor.black.cgColor)
|
||||||
|
context.fill(CGRect(origin: .zero, size: targetSize))
|
||||||
|
context.saveGState()
|
||||||
|
context.concatenate(transform)
|
||||||
|
context.draw(source, in: CGRect(x: 0, y: 0, width: source.width, height: source.height))
|
||||||
|
context.restoreGState()
|
||||||
|
return context.makeImage()!
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func morphPoints(landmarks: [CGPoint]?, targetSize: CGSize) -> [CGPoint] {
|
||||||
|
let w = targetSize.width
|
||||||
|
let h = targetSize.height
|
||||||
|
let boundary = [
|
||||||
|
CGPoint(x: 0, y: 0), CGPoint(x: w * 0.25, y: 0), CGPoint(x: w * 0.5, y: 0), CGPoint(x: w * 0.75, y: 0), CGPoint(x: w, y: 0),
|
||||||
|
CGPoint(x: 0, y: h * 0.25), CGPoint(x: w, y: h * 0.25),
|
||||||
|
CGPoint(x: 0, y: h * 0.5), CGPoint(x: w, y: h * 0.5),
|
||||||
|
CGPoint(x: 0, y: h * 0.75), CGPoint(x: w, y: h * 0.75),
|
||||||
|
CGPoint(x: 0, y: h), CGPoint(x: w * 0.25, y: h), CGPoint(x: w * 0.5, y: h), CGPoint(x: w * 0.75, y: h), CGPoint(x: w, y: h)
|
||||||
|
]
|
||||||
|
|
||||||
|
guard let landmarks, !landmarks.isEmpty else {
|
||||||
|
return boundary
|
||||||
|
}
|
||||||
|
|
||||||
|
let clipped = landmarks.map {
|
||||||
|
CGPoint(x: min(max(0, $0.x), w), y: min(max(0, $0.y), h))
|
||||||
|
}
|
||||||
|
return clipped + boundary
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func renderMorphFrame(
|
||||||
|
left: PreparedMorphImage,
|
||||||
|
right: PreparedMorphImage,
|
||||||
|
triangles: [MorphTriangle],
|
||||||
|
t: Double
|
||||||
|
) throws -> CGImage {
|
||||||
|
let width = left.image.width
|
||||||
|
let height = left.image.height
|
||||||
|
let context = makeContext(width: width, height: height)
|
||||||
|
context.setFillColor(NSColor.black.cgColor)
|
||||||
|
context.fill(CGRect(x: 0, y: 0, width: width, height: height))
|
||||||
|
|
||||||
|
let count = min(left.points.count, right.points.count)
|
||||||
|
let targetPoints = (0..<count).map { index in
|
||||||
|
interpolate(left.points[index], right.points[index], t)
|
||||||
|
}
|
||||||
|
|
||||||
|
for triangle in triangles where triangle.a < count && triangle.b < count && triangle.c < count {
|
||||||
|
let target = [targetPoints[triangle.a], targetPoints[triangle.b], targetPoints[triangle.c]]
|
||||||
|
let leftTriangle = [left.points[triangle.a], left.points[triangle.b], left.points[triangle.c]]
|
||||||
|
let rightTriangle = [right.points[triangle.a], right.points[triangle.b], right.points[triangle.c]]
|
||||||
|
drawWarped(left.image, sourceTriangle: leftTriangle, targetTriangle: target, alpha: CGFloat(1 - t), in: context)
|
||||||
|
drawWarped(right.image, sourceTriangle: rightTriangle, targetTriangle: target, alpha: CGFloat(t), in: context)
|
||||||
|
}
|
||||||
|
|
||||||
|
guard let image = context.makeImage() else {
|
||||||
|
throw VideoProcessingError.invalidSettings("Morph-Frame konnte nicht erzeugt werden.")
|
||||||
|
}
|
||||||
|
return image
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func drawWarped(
|
||||||
|
_ image: CGImage,
|
||||||
|
sourceTriangle: [CGPoint],
|
||||||
|
targetTriangle: [CGPoint],
|
||||||
|
alpha: CGFloat,
|
||||||
|
in context: CGContext
|
||||||
|
) {
|
||||||
|
guard alpha > 0.001,
|
||||||
|
let transform = affineTransform(from: sourceTriangle, to: targetTriangle) else {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
context.saveGState()
|
||||||
|
let path = CGMutablePath()
|
||||||
|
path.move(to: targetTriangle[0])
|
||||||
|
path.addLine(to: targetTriangle[1])
|
||||||
|
path.addLine(to: targetTriangle[2])
|
||||||
|
path.closeSubpath()
|
||||||
|
context.addPath(path)
|
||||||
|
context.clip()
|
||||||
|
context.setAlpha(alpha)
|
||||||
|
context.concatenate(transform)
|
||||||
|
context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))
|
||||||
|
context.restoreGState()
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func triangulate(_ points: [CGPoint]) -> [MorphTriangle] {
|
||||||
|
guard points.count >= 3 else {
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
|
||||||
|
var triangles = Set<MorphTriangle>()
|
||||||
|
for a in 0..<(points.count - 2) {
|
||||||
|
for b in (a + 1)..<(points.count - 1) {
|
||||||
|
for c in (b + 1)..<points.count {
|
||||||
|
guard abs(area(points[a], points[b], points[c])) > 0.5 else {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
guard let circle = circumcircle(points[a], points[b], points[c]) else {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var containsPoint = false
|
||||||
|
for index in 0..<points.count where index != a && index != b && index != c {
|
||||||
|
let distance = hypot(points[index].x - circle.center.x, points[index].y - circle.center.y)
|
||||||
|
if distance < circle.radius - 0.01 {
|
||||||
|
containsPoint = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !containsPoint {
|
||||||
|
triangles.insert(MorphTriangle(a: a, b: b, c: c))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Array(triangles)
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func area(_ a: CGPoint, _ b: CGPoint, _ c: CGPoint) -> CGFloat {
|
||||||
|
(b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x)
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func circumcircle(_ a: CGPoint, _ b: CGPoint, _ c: CGPoint) -> (center: CGPoint, radius: CGFloat)? {
|
||||||
|
let d = 2 * (a.x * (b.y - c.y) + b.x * (c.y - a.y) + c.x * (a.y - b.y))
|
||||||
|
guard abs(d) > 0.0001 else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
let a2 = a.x * a.x + a.y * a.y
|
||||||
|
let b2 = b.x * b.x + b.y * b.y
|
||||||
|
let c2 = c.x * c.x + c.y * c.y
|
||||||
|
let ux = (a2 * (b.y - c.y) + b2 * (c.y - a.y) + c2 * (a.y - b.y)) / d
|
||||||
|
let uy = (a2 * (c.x - b.x) + b2 * (a.x - c.x) + c2 * (b.x - a.x)) / d
|
||||||
|
let center = CGPoint(x: ux, y: uy)
|
||||||
|
return (center, hypot(center.x - a.x, center.y - a.y))
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func affineTransform(from source: [CGPoint], to target: [CGPoint]) -> CGAffineTransform? {
|
||||||
|
let x1 = source[0].x, y1 = source[0].y
|
||||||
|
let x2 = source[1].x, y2 = source[1].y
|
||||||
|
let x3 = source[2].x, y3 = source[2].y
|
||||||
|
let u1 = target[0].x, v1 = target[0].y
|
||||||
|
let u2 = target[1].x, v2 = target[1].y
|
||||||
|
let u3 = target[2].x, v3 = target[2].y
|
||||||
|
|
||||||
|
let determinant = x1 * (y2 - y3) + x2 * (y3 - y1) + x3 * (y1 - y2)
|
||||||
|
guard abs(determinant) > 0.0001 else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
let a = (u1 * (y2 - y3) + u2 * (y3 - y1) + u3 * (y1 - y2)) / determinant
|
||||||
|
let c = (u1 * (x3 - x2) + u2 * (x1 - x3) + u3 * (x2 - x1)) / determinant
|
||||||
|
let tx = (u1 * (x2 * y3 - x3 * y2) + u2 * (x3 * y1 - x1 * y3) + u3 * (x1 * y2 - x2 * y1)) / determinant
|
||||||
|
let b = (v1 * (y2 - y3) + v2 * (y3 - y1) + v3 * (y1 - y2)) / determinant
|
||||||
|
let d = (v1 * (x3 - x2) + v2 * (x1 - x3) + v3 * (x2 - x1)) / determinant
|
||||||
|
let ty = (v1 * (x2 * y3 - x3 * y2) + v2 * (x3 * y1 - x1 * y3) + v3 * (x1 * y2 - x2 * y1)) / determinant
|
||||||
|
|
||||||
|
return CGAffineTransform(a: a, b: b, c: c, d: d, tx: tx, ty: ty)
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func pointInImage(_ point: CGPoint, face: VNFaceObservation, image: CGImage) -> CGPoint {
|
||||||
|
let x = (face.boundingBox.minX + point.x * face.boundingBox.width) * CGFloat(image.width)
|
||||||
|
let y = (1 - (face.boundingBox.minY + point.y * face.boundingBox.height)) * CGFloat(image.height)
|
||||||
|
return CGPoint(x: x, y: y)
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func faceScore(_ face: VNFaceObservation) -> CGFloat {
|
||||||
|
let area = face.boundingBox.width * face.boundingBox.height
|
||||||
|
let center = hypot(face.boundingBox.midX - 0.5, face.boundingBox.midY - 0.5)
|
||||||
|
return area - center * 0.08
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func interpolate(_ left: CGPoint, _ right: CGPoint, _ t: Double) -> CGPoint {
|
||||||
|
CGPoint(
|
||||||
|
x: left.x + (right.x - left.x) * CGFloat(t),
|
||||||
|
y: left.y + (right.y - left.y) * CGFloat(t)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func makeContext(width: Int, height: Int) -> CGContext {
|
||||||
|
let colorSpace = CGColorSpace(name: CGColorSpace.sRGB)!
|
||||||
|
let context = CGContext(
|
||||||
|
data: nil,
|
||||||
|
width: width,
|
||||||
|
height: height,
|
||||||
|
bitsPerComponent: 8,
|
||||||
|
bytesPerRow: width * 4,
|
||||||
|
space: colorSpace,
|
||||||
|
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
|
||||||
|
)!
|
||||||
|
return context
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func writePNG(_ image: CGImage, to url: URL) throws {
|
||||||
|
guard let destination = CGImageDestinationCreateWithURL(url as CFURL, UTType.png.identifier as CFString, 1, nil) else {
|
||||||
|
throw VideoProcessingError.invalidSettings("PNG konnte nicht geschrieben werden: \(url.lastPathComponent)")
|
||||||
|
}
|
||||||
|
CGImageDestinationAddImage(destination, image, nil)
|
||||||
|
guard CGImageDestinationFinalize(destination) else {
|
||||||
|
throw VideoProcessingError.invalidSettings("PNG konnte nicht finalisiert werden: \(url.lastPathComponent)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private struct BestSegmentResult {
|
private struct BestSegmentResult {
|
||||||
let start: Double
|
let start: Double
|
||||||
let score: Double
|
let score: Double
|
||||||
|
|||||||
Reference in New Issue
Block a user