// // Copyright 2025 Element Creations Ltd. // Copyright 2023-2025 New Vector Ltd. // // SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial. // Please see LICENSE files in the repository root for full details. // import AVFoundation import MatrixRustSDK import SwiftUI import UniformTypeIdentifiers indirect nonisolated enum MediaUploadingPreprocessorError: Error { case cancelled case maxUploadSizeUnknown case maxUploadSizeExceeded(limit: UInt) case failedProcessingMedia(Error) case failedProcessingImage(MediaUploadingPreprocessorError) case failedProcessingVideo(MediaUploadingPreprocessorError) case failedProcessingAudio case failedGeneratingVideoThumbnail(Error?) case failedGeneratingImageThumbnail(Error?) case failedStrippingLocationData case failedResizingImage case failedConvertingVideo } nonisolated enum MediaInfo { case image(imageURL: URL, thumbnailURL: URL, imageInfo: ImageInfo) case video(videoURL: URL, thumbnailURL: URL, videoInfo: VideoInfo) case audio(audioURL: URL, audioInfo: AudioInfo) case file(fileURL: URL, fileInfo: FileInfo) var mimeType: String? { switch self { case .image(_, _, let imageInfo): return imageInfo.mimetype case .video(_, _, let videoInfo): return videoInfo.mimetype case .audio(_, let audioInfo): return audioInfo.mimetype case .file(_, let fileInfo): return fileInfo.mimetype } } var url: URL { switch self { case .image(let url, _, _), .video(let url, _, _), .audio(let url, _), .file(let url, _): return url } } var thumbnailURL: URL? { switch self { case .image(_, let url, _), .video(_, let url, _): return url case .audio, .file: return nil } } } private nonisolated struct ImageProcessingInfo { let url: URL let height: Double let width: Double let mimeType: String let blurhash: String? } private nonisolated struct VideoProcessingInfo { let url: URL let height: Double let width: Double let duration: Double // seconds let mimeType: String } nonisolated struct MediaUploadingPreprocessor { let userSettings: UserSettings nonisolated enum Constants { static let maximumThumbnailSize = CGSize(width: 800, height: 600) static let optimizedMaxPixelSize = 2048.0 static let jpegCompressionQuality = 0.78 static let videoThumbnailTime = 5.0 // seconds static let maximumConcurrentProcessingCount = 2 } /// Processes media at the given URLs. It will generate thumbnails for images and videos, convert videos to 1080p mp4, strip GPS locations /// from images and retrieve associated media information. On failure or cancellation all the prepared files are removed. /// - Parameter urls: the file URLs /// - Returns: a collection of results containing specific type of `MediaInfo` depending on the file type /// and its associated details or any resulting error func processMedia(at urls: [URL], maxUploadSize: UInt) async -> Result<[MediaInfo], MediaUploadingPreprocessorError> { await withTaskGroup(of: (Int, Result).self) { taskGroup in var pendingIndices = urls.indices.makeIterator() func processNextURL() { guard let index = pendingIndices.next() else { return } _ = taskGroup.addTaskUnlessCancelled { await (index, processMedia(at: urls[index], maxUploadSize: maxUploadSize)) } } for _ in 0.. Result { guard !Task.isCancelled else { return .failure(.cancelled) } let preparedURL: URL do { preparedURL = try await copyToUniqueFolder(url) } catch { return .failure(error) } let result = await processPreparedMedia(at: preparedURL, maxUploadSize: maxUploadSize) if Task.isCancelled { await removeUniqueFolders(containing: [preparedURL]) return .failure(.cancelled) } if case .failure = result { await removeUniqueFolders(containing: [preparedURL]) } return result } // MARK: - Private /// Copies the file into a new unique folder to avoid conflicts when processing the same file more than once. /// Every file derived from the copy (converted media, thumbnails) is written into that same folder. @concurrent private func copyToUniqueFolder(_ url: URL) async throws(MediaUploadingPreprocessorError) -> URL { let uniqueFolder = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) let newURL = uniqueFolder.appendingPathComponent(url.lastPathComponent) do { try FileManager.default.createDirectory(at: uniqueFolder, withIntermediateDirectories: true) try FileManager.default.copyItem(at: url, to: newURL) return newURL } catch { try? FileManager.default.removeItem(at: uniqueFolder) throw .failedProcessingMedia(error) } } /// Removes the whole unique folder (see `copyToUniqueFolder`) that holds each of the given files. @concurrent private func removeUniqueFolders(containing urls: [URL]) async { for url in urls { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) } } private func processPreparedMedia(at url: URL, maxUploadSize: UInt) async -> Result { guard !Task.isCancelled else { return .failure(.cancelled) } // Process unknown types as plain files guard let type = UTType(filenameExtension: url.pathExtension), let mimeType = type.preferredMIMEType else { return processFile(at: url, mimeType: "application/octet-stream", maxUploadSize: maxUploadSize) } if type.conforms(to: .image) { return await processImage(at: url, type: type, mimeType: mimeType, maxUploadSize: maxUploadSize) } else if type.conforms(to: .movie) || type.conforms(to: .video) { return await processVideo(at: url, maxUploadSize: maxUploadSize) } else if type.conforms(to: .audio) { return await processAudio(at: url, mimeType: mimeType, maxUploadSize: maxUploadSize) } else { return processFile(at: url, mimeType: mimeType, maxUploadSize: maxUploadSize) } } /// Prepares an image for upload. Strips location data from it and generates a thumbnail /// - Parameters: /// - url: The image URL /// - type: its UTType /// - mimeType: the mimeType extracted from the UTType /// - Returns: Returns a `MediaInfo.image` containing the URLs for the modified image and its thumbnail plus the corresponding `ImageInfo` @concurrent private func processImage(at url: URL, type: UTType, mimeType: String, maxUploadSize: UInt) async -> Result { var url = url do { try stripLocationFromImage(at: url, type: type) guard !Task.isCancelled else { return .failure(.cancelled) } var mimeType = mimeType if userSettings.app.optimizeMediaUploads, !type.conforms(to: .gif) { let outputType = type.conforms(to: .png) ? UTType.png : .jpeg mimeType = outputType.preferredMIMEType ?? "application/octet-stream" try resizeImage(at: url, maxPixelSize: Constants.optimizedMaxPixelSize, destination: url, type: outputType) if let preferredFilenameExtension = outputType.preferredFilenameExtension, url.pathExtension != preferredFilenameExtension { let convertedURL = url.deletingPathExtension().appendingPathExtension(preferredFilenameExtension) do { try FileManager.default.moveItem(at: url, to: convertedURL) } catch { return .failure(.failedResizingImage) } url = convertedURL } } guard !Task.isCancelled else { return .failure(.cancelled) } let thumbnailResult = try generateThumbnailForImage(at: url) guard let imageSource = CGImageSourceCreateWithURL(url as NSURL, nil), let imageSize = imageSource.size else { return .failure(.failedProcessingImage(.failedStrippingLocationData)) } let fileSize = (try? FileManager.default.sizeForItem(at: url)) ?? 0 let thumbnailFileSize = (try? FileManager.default.sizeForItem(at: thumbnailResult.url)) ?? 0 guard fileSize < maxUploadSize, thumbnailFileSize < maxUploadSize else { return .failure(.maxUploadSizeExceeded(limit: maxUploadSize)) } let thumbnailInfo = ThumbnailInfo(height: UInt64(thumbnailResult.height), width: UInt64(thumbnailResult.width), mimetype: thumbnailResult.mimeType, size: UInt64(thumbnailFileSize)) let imageInfo = ImageInfo(height: UInt64(imageSize.height), width: UInt64(imageSize.width), mimetype: mimeType, size: UInt64(fileSize), thumbnailInfo: thumbnailInfo, thumbnailSource: nil, blurhash: thumbnailResult.blurhash, isAnimated: nil) let mediaInfo = MediaInfo.image(imageURL: url, thumbnailURL: thumbnailResult.url, imageInfo: imageInfo) return .success(mediaInfo) } catch { return .failure(.failedProcessingImage(error)) } } /// Prepares a video for upload. Converts it to an 1080p mp4 and generates a thumbnail /// - Parameters: /// - url: The video URL /// - type: its UTType /// - mimeType: the mimeType extracted from the UTType /// - Returns: Returns a `MediaInfo.video` containing the URLs for the modified video and its thumbnail plus the corresponding `VideoInfo` private func processVideo(at url: URL, maxUploadSize: UInt) async -> Result { do { let result = try await convertVideoToMP4(url, targetFileSize: UInt(maxUploadSize)) guard !Task.isCancelled else { return .failure(.cancelled) } let thumbnailResult = try await generateThumbnailForVideo(at: result.url) let videoSize = (try? FileManager.default.sizeForItem(at: result.url)) ?? 0 let thumbnailSize = (try? FileManager.default.sizeForItem(at: thumbnailResult.url)) ?? 0 guard videoSize < maxUploadSize, thumbnailSize < maxUploadSize else { return .failure(.maxUploadSizeExceeded(limit: maxUploadSize)) } let thumbnailInfo = ThumbnailInfo(height: UInt64(thumbnailResult.height), width: UInt64(thumbnailResult.width), mimetype: thumbnailResult.mimeType, size: UInt64(thumbnailSize)) let videoInfo = VideoInfo(duration: result.duration, height: UInt64(result.height), width: UInt64(result.width), mimetype: result.mimeType, size: UInt64(videoSize), thumbnailInfo: thumbnailInfo, thumbnailSource: nil, blurhash: thumbnailResult.blurhash) let mediaInfo = MediaInfo.video(videoURL: result.url, thumbnailURL: thumbnailResult.url, videoInfo: videoInfo) return .success(mediaInfo) } catch { return .failure(.failedProcessingVideo(error)) } } /// Prepares a file for upload. /// - Parameters: /// - url: The audio URL /// - mimeType: the mimeType extracted from the UTType /// - Returns: Returns a `MediaInfo.audio` containing the file URL plus the corresponding `AudioInfo` private func processAudio(at url: URL, mimeType: String?, maxUploadSize: UInt) async -> Result { let fileSize = (try? FileManager.default.sizeForItem(at: url)) ?? 0 guard fileSize < maxUploadSize else { return .failure(.maxUploadSizeExceeded(limit: maxUploadSize)) } let asset = AVURLAsset(url: url) guard let durationInSeconds = try? await asset.load(.duration).seconds else { return .failure(.failedProcessingAudio) } let audioInfo = AudioInfo(duration: durationInSeconds, size: UInt64(fileSize), mimetype: mimeType) return .success(.audio(audioURL: url, audioInfo: audioInfo)) } /// Prepares a file for upload. /// - Parameters: /// - url: The file URL /// - type: its UTType /// - mimeType: the mimeType extracted from the UTType /// - Returns: Returns a `MediaInfo.file` containing the file URL plus the corresponding `FileInfo` private func processFile(at url: URL, mimeType: String?, maxUploadSize: UInt) -> Result { let fileSize = (try? FileManager.default.sizeForItem(at: url)) ?? 0 guard fileSize < maxUploadSize else { return .failure(.maxUploadSizeExceeded(limit: maxUploadSize)) } let fileInfo = FileInfo(mimetype: mimeType, size: UInt64(fileSize), thumbnailInfo: nil, thumbnailSource: nil) return .success(.file(fileURL: url, fileInfo: fileInfo)) } // MARK: Image Helpers /// Removes the GPS dictionary from an image's metadata /// - Parameters: /// - url: the URL for the original image /// - type: its UTType /// - Returns: the URL for the modified image and its size as an `ImageProcessingResult` private func stripLocationFromImage(at url: URL, type: UTType) throws(MediaUploadingPreprocessorError) { guard let originalData = NSData(contentsOf: url), let imageSource = CGImageSourceCreateWithData(originalData, nil) else { throw .failedStrippingLocationData } guard let originalMetadata = CGImageSourceCopyPropertiesAtIndex(imageSource, 0, nil), (originalMetadata as NSDictionary).value(forKeyPath: "\(kCGImagePropertyGPSDictionary)") != nil else { MXLog.info("No GPS metadata found. Nothing to do.") return } let count = CGImageSourceGetCount(imageSource) let metadataKeysToRemove = [kCGImagePropertyGPSDictionary: kCFNull] let data = NSMutableData() // Certain type identifiers cannot be used for image destinations, fall // back to `public.jpeg` when that's the case let destination = CGImageDestinationCreateWithData(data as CFMutableData, type.identifier as CFString, count, nil) ?? CGImageDestinationCreateWithData(data as CFMutableData, UTType.jpeg.identifier as CFString, count, nil) guard let destination else { throw .failedStrippingLocationData } CGImageDestinationAddImageFromSource(destination, imageSource, 0, metadataKeysToRemove as NSDictionary) CGImageDestinationFinalize(destination) do { try data.write(to: url) } catch { throw .failedStrippingLocationData } } /// Generates a thumbnail for an image /// - Parameter url: the original image URL /// - Returns: the URL for the resulting thumbnail and its sizing info as an `ImageProcessingResult` private func generateThumbnailForImage(at url: URL) throws(MediaUploadingPreprocessorError) -> ImageProcessingInfo { let thumbnailFileName = "thumbnail-\((url.lastPathComponent as NSString).deletingPathExtension).jpeg" let thumbnailURL = url.deletingLastPathComponent().appendingPathComponent(thumbnailFileName) let thumbnailMaxPixelSize = max(Constants.maximumThumbnailSize.height, Constants.maximumThumbnailSize.width) do { try resizeImage(at: url, maxPixelSize: thumbnailMaxPixelSize, destination: thumbnailURL, type: .jpeg) } catch { throw .failedGeneratingImageThumbnail(error) } guard let thumbnail = UIImage(contentsOfFile: thumbnailURL.path(percentEncoded: false)) else { throw .failedGeneratingImageThumbnail(nil) } let blurhash = thumbnail.blurHash(numberOfComponents: (3, 3)) return .init(url: thumbnailURL, height: thumbnail.size.height, width: thumbnail.size.width, mimeType: "image/jpeg", blurhash: blurhash) } private func resizeImage(at url: URL, maxPixelSize: CGFloat, destination: URL, type: UTType) throws(MediaUploadingPreprocessorError) { guard let imageSource = CGImageSourceCreateWithURL(url as NSURL, nil) else { throw .failedResizingImage } try resizeImage(withSource: imageSource, maxPixelSize: maxPixelSize, destination: destination, type: type) } /// Aspect ratio resizes an image so it fits in the given size. This is useful for resizing images without loading them directly into memory /// - Parameters: /// - imageSource: the original image `CGImageSource` /// - maxPixelSize: maximum resulting size for the largest dimension of the image. /// - Returns: the resized image private func resizeImage(withSource imageSource: CGImageSource, maxPixelSize: CGFloat, destination destinationURL: URL, type: UTType) throws(MediaUploadingPreprocessorError) { let options: [NSString: Any] = [ // The maximum width and height in pixels of a thumbnail. kCGImageSourceThumbnailMaxPixelSize: maxPixelSize, kCGImageSourceCreateThumbnailFromImageAlways: true, // Should include kCGImageSourceCreateThumbnailWithTransform: true in the options dictionary. Otherwise, the image result will appear rotated when an image is taken from camera in the portrait orientation. kCGImageSourceCreateThumbnailWithTransform: true ] guard let scaledImage = CGImageSourceCreateThumbnailAtIndex(imageSource, 0, options as NSDictionary), let destination = CGImageDestinationCreateWithURL(destinationURL as CFURL, type.identifier as CFString, 1, nil) else { throw .failedResizingImage } let properties = [kCGImageDestinationLossyCompressionQuality: Constants.jpegCompressionQuality] CGImageDestinationAddImage(destination, scaledImage, properties as NSDictionary) CGImageDestinationFinalize(destination) } // MARK: Video Helpers /// Generates a thumbnail for the video at the given URL /// - Parameter url: the video URL /// - Returns: the URL for the resulting thumbnail and its sizing info as an `ImageProcessingResult` @concurrent private func generateThumbnailForVideo(at url: URL) async throws(MediaUploadingPreprocessorError) -> ImageProcessingInfo { let assetImageGenerator = AVAssetImageGenerator(asset: AVURLAsset(url: url)) assetImageGenerator.appliesPreferredTrackTransform = true assetImageGenerator.maximumSize = Constants.maximumThumbnailSize // Avoid the first frames as on a lot of videos they're black. // If the specified seconds are longer than the actual video a frame close to the end of the video will be used, at AVFoundation's discretion let location = CMTime(seconds: Constants.videoThumbnailTime, preferredTimescale: 1) let cgImage: CGImage do { cgImage = try await assetImageGenerator.image(at: location).image } catch { throw .failedGeneratingVideoThumbnail(error) } let thumbnail = UIImage(cgImage: cgImage) guard let data = thumbnail.jpegData(compressionQuality: Constants.jpegCompressionQuality) else { throw .failedGeneratingVideoThumbnail(nil) } let blurhash = thumbnail.blurHash(numberOfComponents: (3, 3)) let fileName = "\((url.lastPathComponent as NSString).deletingPathExtension).jpeg" let thumbnailURL = url.deletingLastPathComponent().appendingPathComponent(fileName) do { try data.write(to: thumbnailURL) } catch { throw .failedGeneratingVideoThumbnail(error) } return .init(url: thumbnailURL, height: thumbnail.size.height, width: thumbnail.size.width, mimeType: "image/jpeg", blurhash: blurhash) } /// Converts the given video to an 1080p mp4 /// - Parameters: /// - url: the original video URL /// - targetFileSize: the maximum resulting file size. 90% of this will be used /// - Returns: the URL for the resulting video and its media info as a `VideoProcessingResult` private func convertVideoToMP4(_ url: URL, targetFileSize: UInt) async throws(MediaUploadingPreprocessorError) -> VideoProcessingInfo { let asset = AVURLAsset(url: url) let presetName = userSettings.app.optimizeMediaUploads ? AVAssetExportPreset1280x720 : AVAssetExportPreset1920x1080 guard let exportSession = AVAssetExportSession(asset: asset, presetName: presetName) else { throw .failedConvertingVideo } // AVAssetExportSession will fail if the output URL already exists let uuid = UUID().uuidString let originalFilenameWithoutExtension = url.deletingPathExtension().lastPathComponent let outputURL = url.deletingLastPathComponent().appendingPathComponent("\(uuid)-\(originalFilenameWithoutExtension).mp4") try? FileManager.default.removeItem(at: outputURL) guard exportSession.supportedFileTypes.contains(AVFileType.mp4) else { throw .failedConvertingVideo } if targetFileSize > 0 { // Reduce the target file size by 10% as fileLengthLimit isn't a hard limit exportSession.fileLengthLimit = Int64(Double(targetFileSize) * 0.9) } do { try await exportSession.export(to: outputURL, as: .mp4) } catch { guard !Task.isCancelled else { throw .cancelled } MXLog.error("Video conversion failed: \(error)") throw .failedConvertingVideo } // Delete the original try? FileManager.default.removeItem(at: url) // Strip the UUID from the new version let newOutputURL = url.deletingLastPathComponent().appendingPathComponent("\(originalFilenameWithoutExtension).mp4") do { try FileManager.default.moveItem(at: outputURL, to: newOutputURL) } catch { throw .failedConvertingVideo } let newAsset = AVURLAsset(url: newOutputURL) guard let track = try? await newAsset.loadTracks(withMediaType: .video).first, let durationInSeconds = try? await newAsset.load(.duration).seconds, let adjustedNaturalSize = try? await track.size else { throw .failedConvertingVideo } return .init(url: newOutputURL, height: adjustedNaturalSize.height, width: adjustedNaturalSize.width, duration: durationInSeconds, mimeType: "video/mp4") } } // MARK: - Extensions private nonisolated extension CGImageSource { var size: CGSize? { guard let properties = CGImageSourceCopyPropertiesAtIndex(self, 0, nil) as? [NSString: Any], var width = properties[kCGImagePropertyPixelWidth] as? Int, var height = properties[kCGImagePropertyPixelHeight] as? Int else { return nil } // Make sure the width and height are the correct way around if an orientation is set. if let orientationValue = properties[kCGImagePropertyOrientation] as? UInt32, let orientation = CGImagePropertyOrientation(rawValue: orientationValue) { switch orientation { case .up, .down, .upMirrored, .downMirrored: break case .left, .right, .leftMirrored, .rightMirrored: swap(&width, &height) } } return CGSize(width: width, height: height) } } private nonisolated extension AVAssetTrack { var size: CGSize { get async throws { let naturalSize = try await load(.naturalSize) guard mediaType == .video else { return naturalSize } // The naturalSize does not take the preferredTransform into consideration resulting // in portrait videos reporting inverted values. let transform = try await load(.preferredTransform) switch (transform.a, transform.b, transform.c, transform.d) { case (0, 1, -1, 0), (0, -1, 1, 0): return CGSize(width: naturalSize.height, height: naturalSize.width) default: return CGSize(width: naturalSize.width, height: naturalSize.height) } } } }