#748 - Implement methods for processing various media types and prepa… (#772)

* #748 - Implement methods for processing various media types and preparing them for upload

* Various tweaks following code review:
- added blurhash generation
- changed thumbnails to jpeg
- throwing if file size cannot be retrieved
- move MediaProvider files to separate folder so they can cleanly be imported in the NSE
- added audio file processing
- switched all image resizing methods to ImageIO
- various renames

* Fix blurhash formatting issues, `swift-format-ignore-file` doesn't work

* Processing all media within unique folders to avoid conflicts

* Fix various warnings

* Rename `sizeForItemAt(_ url: URL)` to `sizeForItem(at url: URL)`

* Remove unnecessary resizeImage(UIImage) method, start by copying not moving the file to the unique location

* Add back warning for roomDetails.avatarURL

ElementX/Sources/Services/Media/Provider/ImageProviderProtocol.swift

@@ -0,0 +1,33 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import UIKit
+
+protocol ImageProviderProtocol {
+    func imageFromSource(_ source: MediaSourceProxy?, size: CGSize?) -> UIImage?
+    
+    func loadImageFromSource(_ source: MediaSourceProxy, size: CGSize?) async -> Result<UIImage, MediaProviderError>
+}
+
+extension ImageProviderProtocol {
+    func imageFromSource(_ source: MediaSourceProxy?) -> UIImage? {
+        imageFromSource(source, size: nil)
+    }
+    
+    func loadImageFromSource(_ source: MediaSourceProxy) async -> Result<UIImage, MediaProviderError> {
+        await loadImageFromSource(source, size: nil)
+    }
+}

ElementX/Sources/Services/Media/Provider/MediaFileHandleProxy.swift

@@ -0,0 +1,68 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+import MatrixRustSDK
+
+/// A wrapper around Rust's `MediaFileHandle` type that provides us with a
+/// media file that is stored unencrypted in a temporary location for previewing.
+class MediaFileHandleProxy {
+    /// The underlying handle for the file.
+    private let handle: MediaFileHandleProtocol
+    
+    /// Creates a new instance from the Rust type.
+    init(handle: MediaFileHandleProtocol) {
+        self.handle = handle
+    }
+    
+    /// Creates an unmanaged instance (for mocking etc), using a raw `URL`
+    ///
+    /// A media file created from a URL won't have the automatic clean-up mechanism
+    /// that is provided by the SDK's `MediaFileHandle`.
+    static func unmanaged(url: URL) -> MediaFileHandleProxy {
+        MediaFileHandleProxy(handle: UnmanagedMediaFileHandle(url: url))
+    }
+    
+    /// The media file's location on disk.
+    var url: URL {
+        URL(filePath: handle.path())
+    }
+}
+
+// MARK: - Hashable
+
+extension MediaFileHandleProxy: Hashable {
+    static func == (lhs: MediaFileHandleProxy, rhs: MediaFileHandleProxy) -> Bool {
+        lhs.url == rhs.url
+    }
+    
+    func hash(into hasher: inout Hasher) {
+        hasher.combine(url)
+    }
+}
+
+// MARK: -
+
+/// An unmanaged file handle that can be created direct from a URL.
+///
+/// This type allows for mocking but doesn't provide the automatic clean-up mechanism provided by the SDK.
+private struct UnmanagedMediaFileHandle: MediaFileHandleProtocol {
+    let url: URL
+    
+    func path() -> String {
+        url.path()
+    }
+}

ElementX/Sources/Services/Media/Provider/MediaLoader.swift

@@ -0,0 +1,88 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+import Foundation
+import MatrixRustSDK
+import UIKit
+
+private final class MediaRequest {
+    var continuations: [CheckedContinuation<Data, Error>] = []
+}
+
+actor MediaLoader: MediaLoaderProtocol {
+    private let client: ClientProtocol
+    private let clientQueue: DispatchQueue
+    private var ongoingRequests = [MediaSourceProxy: MediaRequest]()
+
+    init(client: ClientProtocol,
+         clientQueue: DispatchQueue = .global()) {
+        self.client = client
+        self.clientQueue = clientQueue
+    }
+    
+    func loadMediaContentForSource(_ source: MediaSourceProxy) async throws -> Data {
+        try await enqueueLoadMediaRequest(forSource: source) {
+            try self.client.getMediaContent(mediaSource: source.underlyingSource)
+        }
+    }
+    
+    func loadMediaThumbnailForSource(_ source: MediaSourceProxy, width: UInt, height: UInt) async throws -> Data {
+        try await enqueueLoadMediaRequest(forSource: source) {
+            try self.client.getMediaThumbnail(mediaSource: source.underlyingSource, width: UInt64(width), height: UInt64(height))
+        }
+    }
+    
+    func loadMediaFileForSource(_ source: MediaSourceProxy) async throws -> MediaFileHandleProxy {
+        let result = try await Task.dispatch(on: clientQueue) {
+            try self.client.getMediaFile(source: source.underlyingSource, mimeType: source.mimeType ?? "application/octet-stream")
+        }
+        
+        return MediaFileHandleProxy(handle: result)
+    }
+    
+    // MARK: - Private
+    
+    private func enqueueLoadMediaRequest(forSource source: MediaSourceProxy, operation: @escaping () throws -> [UInt8]) async throws -> Data {
+        if let ongoingRequest = ongoingRequests[source] {
+            return try await withCheckedThrowingContinuation { continuation in
+                ongoingRequest.continuations.append(continuation)
+            }
+        }
+        
+        let ongoingRequest = MediaRequest()
+        ongoingRequests[source] = ongoingRequest
+        
+        defer {
+            ongoingRequests[source] = nil
+        }
+        
+        do {
+            let result = try await Task.dispatch(on: clientQueue) {
+                let bytes = try operation()
+                return Data(bytes: bytes, count: bytes.count)
+            }
+            
+            ongoingRequest.continuations.forEach { $0.resume(returning: result) }
+            
+            return result
+            
+        } catch {
+            ongoingRequest.continuations.forEach { $0.resume(throwing: error) }
+            throw error
+        }
+    }
+}

ElementX/Sources/Services/Media/Provider/MediaLoaderProtocol.swift

@@ -0,0 +1,25 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+
+protocol MediaLoaderProtocol {
+    func loadMediaContentForSource(_ source: MediaSourceProxy) async throws -> Data
+
+    func loadMediaThumbnailForSource(_ source: MediaSourceProxy, width: UInt, height: UInt) async throws -> Data
+    
+    func loadMediaFileForSource(_ source: MediaSourceProxy) async throws -> MediaFileHandleProxy
+}

ElementX/Sources/Services/Media/Provider/MediaProvider.swift

@@ -0,0 +1,116 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Kingfisher
+import UIKit
+
+struct MediaProvider: MediaProviderProtocol {
+    private let mediaLoader: MediaLoaderProtocol
+    private let imageCache: Kingfisher.ImageCache
+    private let backgroundTaskService: BackgroundTaskServiceProtocol?
+    
+    init(mediaLoader: MediaLoaderProtocol,
+         imageCache: Kingfisher.ImageCache,
+         backgroundTaskService: BackgroundTaskServiceProtocol?) {
+        self.mediaLoader = mediaLoader
+        self.imageCache = imageCache
+        self.backgroundTaskService = backgroundTaskService
+    }
+    
+    // MARK: Images
+    
+    func imageFromSource(_ source: MediaSourceProxy?, size: CGSize?) -> UIImage? {
+        guard let url = source?.url else {
+            return nil
+        }
+        let cacheKey = cacheKeyForURL(url, size: size)
+        return imageCache.retrieveImageInMemoryCache(forKey: cacheKey, options: nil)
+    }
+    
+    func loadImageFromSource(_ source: MediaSourceProxy, size: CGSize?) async -> Result<UIImage, MediaProviderError> {
+        if let image = imageFromSource(source, size: size) {
+            return .success(image)
+        }
+        
+        let loadImageBgTask = await backgroundTaskService?.startBackgroundTask(withName: "LoadImage: \(source.url.hashValue)")
+        defer {
+            loadImageBgTask?.stop()
+        }
+        
+        let cacheKey = cacheKeyForURL(source.url, size: size)
+
+        if case let .success(cacheResult) = await imageCache.retrieveImage(forKey: cacheKey),
+           let image = cacheResult.image {
+            return .success(image)
+        }
+
+        do {
+            let imageData: Data
+            if let size {
+                imageData = try await mediaLoader.loadMediaThumbnailForSource(source, width: UInt(size.width), height: UInt(size.height))
+            } else {
+                imageData = try await mediaLoader.loadMediaContentForSource(source)
+            }
+
+            guard let image = UIImage(data: imageData) else {
+                MXLog.error("Invalid image data")
+                return .failure(.invalidImageData)
+            }
+
+            imageCache.store(image, forKey: cacheKey)
+
+            return .success(image)
+        } catch {
+            MXLog.error("Failed retrieving image with error: \(error)")
+            return .failure(.failedRetrievingImage)
+        }
+    }
+    
+    // MARK: Files
+    
+    func loadFileFromSource(_ source: MediaSourceProxy) async -> Result<MediaFileHandleProxy, MediaProviderError> {
+        let loadFileBgTask = await backgroundTaskService?.startBackgroundTask(withName: "LoadFile: \(source.url.hashValue)")
+        defer { loadFileBgTask?.stop() }
+        
+        do {
+            let file = try await mediaLoader.loadMediaFileForSource(source)
+            return .success(file)
+        } catch {
+            MXLog.error("Failed retrieving file with error: \(error)")
+            return .failure(.failedRetrievingFile)
+        }
+    }
+    
+    // MARK: - Private
+    
+    private func cacheKeyForURL(_ url: URL, size: CGSize?) -> String {
+        if let size {
+            return "\(url.absoluteString){\(size.width),\(size.height)}"
+        } else {
+            return url.absoluteString
+        }
+    }
+}
+
+private extension ImageCache {
+    func retrieveImage(forKey key: String) async -> Result<ImageCacheResult, KingfisherError> {
+        await withCheckedContinuation { continuation in
+            retrieveImage(forKey: key) { result in
+                continuation.resume(returning: result)
+            }
+        }
+    }
+}

ElementX/Sources/Services/Media/Provider/MediaProviderProtocol.swift

@@ -0,0 +1,28 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+import UIKit
+
+enum MediaProviderError: Error {
+    case failedRetrievingImage
+    case failedRetrievingFile
+    case invalidImageData
+}
+
+protocol MediaProviderProtocol: ImageProviderProtocol {
+    func loadFileFromSource(_ source: MediaSourceProxy) async -> Result<MediaFileHandleProxy, MediaProviderError>
+}

ElementX/Sources/Services/Media/Provider/MediaSourceProxy.swift

@@ -0,0 +1,53 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+import MatrixRustSDK
+
+struct MediaSourceProxy: Hashable {
+    /// The media source provided by Rust.
+    let underlyingSource: MediaSource
+    /// The media's mime type, used when loading the media's file.
+    /// This is optional when loading images and thumbnails in memory.
+    let mimeType: String?
+    
+    init(source: MediaSource, mimeType: String?) {
+        underlyingSource = source
+        self.mimeType = mimeType
+    }
+    
+    init(url: URL, mimeType: String?) {
+        underlyingSource = mediaSourceFromUrl(url: url.absoluteString)
+        self.mimeType = mimeType
+    }
+
+    var url: URL! {
+        // Expecting these to always be valid URLs
+        URL(string: underlyingSource.url())
+    }
+}
+
+// MARK: - Hashable
+
+extension MediaSource: Hashable {
+    public static func == (lhs: MediaSource, rhs: MediaSource) -> Bool {
+        lhs.url() == rhs.url()
+    }
+    
+    public func hash(into hasher: inout Hasher) {
+        hasher.combine(url())
+    }
+}

ElementX/Sources/Services/Media/Provider/MockMediaProvider.swift

@@ -0,0 +1,44 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+import UIKit
+
+struct MockMediaProvider: MediaProviderProtocol {
+    func imageFromSource(_ source: MediaSourceProxy?, size: CGSize?) -> UIImage? {
+        guard source != nil else {
+            return nil
+        }
+        
+        if source?.url == .picturesDirectory {
+            return Asset.Images.appLogo.image
+        }
+        
+        return UIImage(systemName: "photo")
+    }
+    
+    func loadImageFromSource(_ source: MediaSourceProxy, size: CGSize?) async -> Result<UIImage, MediaProviderError> {
+        guard let image = UIImage(systemName: "photo") else {
+            fatalError()
+        }
+        
+        return .success(image)
+    }
+    
+    func loadFileFromSource(_ source: MediaSourceProxy) async -> Result<MediaFileHandleProxy, MediaProviderError> {
+        .failure(.failedRetrievingFile)
+    }
+}