Allow image uploads to be optimised to reduce bandwidth. (#3412)

* Use typed throws for intermediate media upload preprocessing steps. * Remove unnecessary async/await usages. * Resize images when optimizedMediaUploads is enabled. * Reduce the JPEG quality. * Add tests for PNG and HEIC, fix mimetypes for these. * Add special handling for GIFs. * Test the files to make sure their mime types match the info. * Update the filename when converting formats. * Extend test timeout for video encoding.
2026-11-03 06:55:59 +00:00 · 2024-10-16 14:50:24 +01:00
parent 35d49c4d85
commit 6078398ad3
5 changed files with 434 additions and 166 deletions
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba429285ac3d7e29a5167bd7fab9a5b5721a9817afe824f61436520f6e6651ec
+size 127823
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f110a5cd97e0e9b38258833274944e816f36379631eeb6883ddf6babb9bef37
+size 1827200
@@ -5,6 +5,7 @@
 // Please see LICENSE in the repository root for full details.
 //

+import UniformTypeIdentifiers
 import XCTest

@testable import ElementX
@@ -45,6 +46,9 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
    }
    
    func testLandscapeMovVideoProcessing() async {
+        // Allow double the default execution time as we encode the video twice now.
+        executionTimeAllowance = 120
+        
        guard let url = Bundle(for: Self.self).url(forResource: "landscape_test_video.mov", withExtension: nil) else {
            XCTFail("Failed retrieving test asset")
            return
@@ -58,6 +62,7 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
        
        // Check that the file name is preserved
        XCTAssertEqual(videoURL.lastPathComponent, "landscape_test_video.mp4")
+        XCTAssertEqual(videoURL.pathExtension, "mp4", "The file extension should match the container we use.")
        
        // Check that the thumbnail is generated correctly
        guard let thumbnailData = try? Data(contentsOf: thumbnailURL),
@@ -79,7 +84,7 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
        
        XCTAssertNotNil(videoInfo.thumbnailInfo)
        XCTAssertEqual(videoInfo.thumbnailInfo?.mimetype, "image/jpeg")
-        XCTAssertEqual(videoInfo.thumbnailInfo?.size ?? 0, 34206, accuracy: 100)
+        XCTAssertEqual(videoInfo.thumbnailInfo?.size ?? 0, 33611, accuracy: 100)
        XCTAssertEqual(videoInfo.thumbnailInfo?.width, 800)
        XCTAssertEqual(videoInfo.thumbnailInfo?.height, 450)
        
@@ -87,11 +92,13 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
        appSettings.optimizeMediaUploads = true
        
        guard case let .success(optimizedResult) = await mediaUploadingPreprocessor.processMedia(at: url),
-              case let .video(_, _, optimizedVideoInfo) = optimizedResult else {
+              case let .video(optimizedVideoURL, _, optimizedVideoInfo) = optimizedResult else {
            XCTFail("Failed processing asset")
            return
        }
        
+        XCTAssertEqual(optimizedVideoURL.pathExtension, "mp4", "The file extension should match the container we use.")
+        
        // Check optimised video info
        XCTAssertEqual(optimizedVideoInfo.mimetype, "video/mp4")
        XCTAssertEqual(optimizedVideoInfo.blurhash, "K32PJbx^I7jYaebHMvV?o$")
@@ -102,6 +109,9 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
    }

    func testPortraitMp4VideoProcessing() async {
+        // Allow double the default execution time as we encode the video twice now.
+        executionTimeAllowance = 120
+        
        guard let url = Bundle(for: Self.self).url(forResource: "portrait_test_video.mp4", withExtension: nil) else {
            XCTFail("Failed retrieving test asset")
            return
@@ -115,6 +125,7 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
        
        // Check that the file name is preserved
        XCTAssertEqual(videoURL.lastPathComponent, "portrait_test_video.mp4")
+        XCTAssertEqual(videoURL.pathExtension, "mp4", "The file extension should match the container we use.")
        
        // Check that the thumbnail is generated correctly
        guard let thumbnailData = try? Data(contentsOf: thumbnailURL),
@@ -136,7 +147,7 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
        
        XCTAssertNotNil(videoInfo.thumbnailInfo)
        XCTAssertEqual(videoInfo.thumbnailInfo?.mimetype, "image/jpeg")
-        XCTAssertEqual(videoInfo.thumbnailInfo?.size ?? 0, 83220, accuracy: 100)
+        XCTAssertEqual(videoInfo.thumbnailInfo?.size ?? 0, 81515, accuracy: 100)
        XCTAssertEqual(videoInfo.thumbnailInfo?.width, 337)
        XCTAssertEqual(videoInfo.thumbnailInfo?.height, 600)
        
@@ -144,11 +155,13 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
        appSettings.optimizeMediaUploads = true
        
        guard case let .success(optimizedResult) = await mediaUploadingPreprocessor.processMedia(at: url),
-              case let .video(_, _, optimizedVideoInfo) = optimizedResult else {
+              case let .video(optimizedVideoURL, _, optimizedVideoInfo) = optimizedResult else {
            XCTFail("Failed processing asset")
            return
        }
        
+        XCTAssertEqual(optimizedVideoURL.pathExtension, "mp4", "The file extension should match the container we use.")
+        
        // Check optimised video info
        XCTAssertEqual(optimizedVideoInfo.mimetype, "video/mp4")
        XCTAssertEqual(optimizedVideoInfo.blurhash, "K7BDNJD*0L%#sl_2~C9ZE1")
@@ -181,9 +194,27 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
        
        XCTAssertNotNil(imageInfo.thumbnailInfo)
        XCTAssertEqual(imageInfo.thumbnailInfo?.mimetype, "image/jpeg")
-        XCTAssertEqual(imageInfo.thumbnailInfo?.size ?? 0, 89553, accuracy: 100)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.size ?? 0, 87733, accuracy: 100)
        XCTAssertEqual(imageInfo.thumbnailInfo?.width, 800)
        XCTAssertEqual(imageInfo.thumbnailInfo?.height, 344)
+        
+        // Repeat with optimised media setting
+        appSettings.optimizeMediaUploads = true
+        
+        guard case let .success(optimizedResult) = await mediaUploadingPreprocessor.processMedia(at: url),
+              case let .image(optimizedImageURL, thumbnailURL, optimizedImageInfo) = optimizedResult else {
+            XCTFail("Failed processing asset")
+            return
+        }
+        
+        compare(originalImageAt: url, toConvertedImageAt: optimizedImageURL, withThumbnailAt: thumbnailURL)
+        
+        // Check optimised image info
+        XCTAssertEqual(optimizedImageInfo.mimetype, "image/jpeg")
+        XCTAssertEqual(optimizedImageInfo.blurhash, "K%I#.NofkC_4ayaxxujsWB")
+        XCTAssertEqual(optimizedImageInfo.size ?? 0, 524_226, accuracy: 100)
+        XCTAssertEqual(optimizedImageInfo.width, 2048)
+        XCTAssertEqual(optimizedImageInfo.height, 879)
    }
    
    func testPortraitImageProcessing() async {
@@ -202,16 +233,193 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
        
        // Check resulting image info
        XCTAssertEqual(imageInfo.mimetype, "image/jpeg")
-        XCTAssertEqual(imageInfo.blurhash, "KdE:ets+RP^-n*RP%OWAV@")
+        XCTAssertEqual(imageInfo.blurhash, "KdE|0Ls+RP^-n*RP%OWAV@")
        XCTAssertEqual(imageInfo.size ?? 0, 4_414_666, accuracy: 100)
        XCTAssertEqual(imageInfo.width, 3024)
        XCTAssertEqual(imageInfo.height, 4032)
        
        XCTAssertNotNil(imageInfo.thumbnailInfo)
        XCTAssertEqual(imageInfo.thumbnailInfo?.mimetype, "image/jpeg")
-        XCTAssertEqual(imageInfo.thumbnailInfo?.size ?? 0, 264_500, accuracy: 100)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.size ?? 0, 258_914, accuracy: 100)
        XCTAssertEqual(imageInfo.thumbnailInfo?.width, 600)
        XCTAssertEqual(imageInfo.thumbnailInfo?.height, 800)
+        
+        // Repeat with optimised media setting
+        appSettings.optimizeMediaUploads = true
+        
+        guard case let .success(optimizedResult) = await mediaUploadingPreprocessor.processMedia(at: url),
+              case let .image(optimizedImageURL, thumbnailURL, optimizedImageInfo) = optimizedResult else {
+            XCTFail("Failed processing asset")
+            return
+        }
+        
+        compare(originalImageAt: url, toConvertedImageAt: optimizedImageURL, withThumbnailAt: thumbnailURL)
+        
+        // Check optimised image info
+        XCTAssertEqual(optimizedImageInfo.mimetype, "image/jpeg")
+        XCTAssertEqual(optimizedImageInfo.blurhash, "KdE|0Ls+RP^-n*RP%OWAV@")
+        XCTAssertEqual(optimizedImageInfo.size ?? 0, 1_462_937, accuracy: 100)
+        XCTAssertEqual(optimizedImageInfo.width, 1536)
+        XCTAssertEqual(optimizedImageInfo.height, 2048)
+    }
+    
+    func testPNGImageProcessing() async {
+        guard let url = Bundle(for: Self.self).url(forResource: "test_image.png", withExtension: nil) else {
+            XCTFail("Failed retrieving test asset")
+            return
+        }
+        
+        guard case let .success(result) = await mediaUploadingPreprocessor.processMedia(at: url),
+              case let .image(convertedImageURL, _, imageInfo) = result else {
+            XCTFail("Failed processing asset")
+            return
+        }
+        
+        // Make sure the output file matches the image info.
+        XCTAssertEqual(mimeType(from: convertedImageURL), "image/png", "PNGs should always be sent as PNG to preserve the alpha channel.")
+        XCTAssertEqual(convertedImageURL.pathExtension, "png", "The file extension should match the MIME type.")
+        
+        // Check resulting image info
+        XCTAssertEqual(imageInfo.mimetype, "image/png")
+        XCTAssertEqual(imageInfo.blurhash, "K0TSUA~qfQ~qj[fQfQfQfQ")
+        XCTAssertEqual(imageInfo.size ?? 0, 4868, accuracy: 100)
+        XCTAssertEqual(imageInfo.width, 240)
+        XCTAssertEqual(imageInfo.height, 240)
+        
+        XCTAssertNotNil(imageInfo.thumbnailInfo)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.mimetype, "image/jpeg")
+        XCTAssertEqual(imageInfo.thumbnailInfo?.size ?? 0, 1725, accuracy: 100)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.width, 240)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.height, 240)
+        
+        // Repeat with optimised media setting
+        appSettings.optimizeMediaUploads = true
+        
+        guard case let .success(optimizedResult) = await mediaUploadingPreprocessor.processMedia(at: url),
+              case let .image(optimizedImageURL, _, optimizedImageInfo) = optimizedResult else {
+            XCTFail("Failed processing asset")
+            return
+        }
+        
+        // Make sure the output file matches the image info.
+        XCTAssertEqual(mimeType(from: optimizedImageURL), "image/png", "PNGs should always be sent as PNG to preserve the alpha channel.")
+        XCTAssertEqual(optimizedImageURL.pathExtension, "png", "The file extension should match the MIME type.")
+        
+        // Check optimised image info
+        XCTAssertEqual(optimizedImageInfo.mimetype, "image/png")
+        XCTAssertEqual(optimizedImageInfo.blurhash, "K0TSUA~qfQ~qj[fQfQfQfQ")
+        XCTAssertEqual(optimizedImageInfo.size ?? 0, 8199, accuracy: 100)
+        // Assert that resizing didn't upscale to the maxPixelSize.
+        XCTAssertEqual(optimizedImageInfo.width, 240)
+        XCTAssertEqual(optimizedImageInfo.height, 240)
+    }
+    
+    func testHEICImageProcessing() async {
+        guard let url = Bundle(for: Self.self).url(forResource: "test_apple_image.heic", withExtension: nil) else {
+            XCTFail("Failed retrieving test asset")
+            return
+        }
+        
+        guard case let .success(result) = await mediaUploadingPreprocessor.processMedia(at: url),
+              case let .image(convertedImageURL, thumbnailURL, imageInfo) = result else {
+            XCTFail("Failed processing asset")
+            return
+        }
+        
+        compare(originalImageAt: url, toConvertedImageAt: convertedImageURL, withThumbnailAt: thumbnailURL)
+        
+        // Make sure the output file matches the image info.
+        XCTAssertEqual(mimeType(from: convertedImageURL), "image/heic", "Unoptimised HEICs should always be sent as is.")
+        XCTAssertEqual(convertedImageURL.pathExtension, "heic", "The file extension should match the MIME type.")
+        
+        // Check resulting image info
+        XCTAssertEqual(imageInfo.mimetype, "image/heic")
+        XCTAssertEqual(imageInfo.blurhash, "KGD]3ns:T00$kWxFXmt6xv")
+        XCTAssertEqual(imageInfo.size ?? 0, 1_857_833, accuracy: 100)
+        XCTAssertEqual(imageInfo.width, 3024)
+        XCTAssertEqual(imageInfo.height, 4032)
+        
+        XCTAssertNotNil(imageInfo.thumbnailInfo)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.mimetype, "image/jpeg")
+        XCTAssertEqual(imageInfo.thumbnailInfo?.size ?? 0, 218_108, accuracy: 100)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.width, 600)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.height, 800)
+        
+        // Repeat with optimised media setting
+        appSettings.optimizeMediaUploads = true
+        
+        guard case let .success(optimizedResult) = await mediaUploadingPreprocessor.processMedia(at: url),
+              case let .image(optimizedImageURL, thumbnailURL, optimizedImageInfo) = optimizedResult else {
+            XCTFail("Failed processing asset")
+            return
+        }
+        
+        compare(originalImageAt: url, toConvertedImageAt: optimizedImageURL, withThumbnailAt: thumbnailURL)
+        
+        // Make sure the output file matches the image info.
+        XCTAssertEqual(mimeType(from: optimizedImageURL), "image/jpeg", "Optimised HEICs should always be converted to JPEG for compatibility.")
+        XCTAssertEqual(optimizedImageURL.pathExtension, "jpeg", "The file extension should match the MIME type.")
+        
+        // Check optimised image info
+        XCTAssertEqual(optimizedImageInfo.mimetype, "image/jpeg")
+        XCTAssertEqual(optimizedImageInfo.blurhash, "KGD]3ns:T00#kWxFb^s:xv")
+        XCTAssertEqual(optimizedImageInfo.size ?? 0, 1_049_393, accuracy: 100)
+        XCTAssertEqual(optimizedImageInfo.width, 1536)
+        XCTAssertEqual(optimizedImageInfo.height, 2048)
+    }
+    
+    func testGIFImageProcessing() async {
+        guard let url = Bundle(for: Self.self).url(forResource: "test_animated_image.gif", withExtension: nil) else {
+            XCTFail("Failed retrieving test asset")
+            return
+        }
+        guard let originalSize = try? FileManager.default.sizeForItem(at: url), originalSize > 0 else {
+            XCTFail("Failed fetching test asset's original size")
+            return
+        }
+        
+        guard case let .success(result) = await mediaUploadingPreprocessor.processMedia(at: url),
+              case let .image(convertedImageURL, _, imageInfo) = result else {
+            XCTFail("Failed processing asset")
+            return
+        }
+        
+        // Make sure the output file matches the image info.
+        XCTAssertEqual(mimeType(from: convertedImageURL), "image/gif", "GIFs should always be sent as GIF to preserve the animation.")
+        XCTAssertEqual(convertedImageURL.pathExtension, "gif", "The file extension should match the MIME type.")
+        
+        // Check resulting image info
+        XCTAssertEqual(imageInfo.mimetype, "image/gif")
+        XCTAssertEqual(imageInfo.blurhash, "K7SY{qs;%NxuRjof~qozIU")
+        XCTAssertEqual(imageInfo.size ?? 0, UInt64(originalSize), accuracy: 100)
+        XCTAssertEqual(imageInfo.width, 490)
+        XCTAssertEqual(imageInfo.height, 498)
+        
+        XCTAssertNotNil(imageInfo.thumbnailInfo)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.mimetype, "image/jpeg")
+        XCTAssertEqual(imageInfo.thumbnailInfo?.size ?? 0, 29511, accuracy: 100)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.width, 490)
+        XCTAssertEqual(imageInfo.thumbnailInfo?.height, 498)
+        
+        // Repeat with optimised media setting
+        appSettings.optimizeMediaUploads = true
+        
+        guard case let .success(optimizedResult) = await mediaUploadingPreprocessor.processMedia(at: url),
+              case let .image(optimizedImageURL, _, optimizedImageInfo) = optimizedResult else {
+            XCTFail("Failed processing asset")
+            return
+        }
+        
+        // Make sure the output file matches the image info.
+        XCTAssertEqual(mimeType(from: optimizedImageURL), "image/gif", "GIFs should always be sent as GIF to preserve the animation.")
+        XCTAssertEqual(optimizedImageURL.pathExtension, "gif", "The file extension should match the MIME type.")
+        
+        // Ensure optimised image is still the same as the original image.
+        XCTAssertEqual(optimizedImageInfo.mimetype, "image/gif")
+        XCTAssertEqual(optimizedImageInfo.blurhash, "K7SY{qs;%NxuRjof~qozIU")
+        XCTAssertEqual(optimizedImageInfo.size ?? 0, UInt64(originalSize), accuracy: 100)
+        XCTAssertEqual(optimizedImageInfo.width, 490)
+        XCTAssertEqual(optimizedImageInfo.height, 498)
    }
    
    // MARK: - Private
@@ -224,11 +432,16 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
            fatalError()
        }
        
-        // Check that the file name is preserved
-        XCTAssertEqual(originalImageURL.lastPathComponent, convertedImageURL.lastPathComponent)
-        
-        // Check that new image is the same size as the original one
-        XCTAssertEqual(originalImage.size, convertedImage.size)
+        if appSettings.optimizeMediaUploads {
+            // Check that new image has been scaled within the requirements for an optimised image
+            XCTAssert(convertedImage.size.width <= MediaUploadingPreprocessor.Constants.optimizedMaxPixelSize)
+            XCTAssert(convertedImage.size.height <= MediaUploadingPreprocessor.Constants.optimizedMaxPixelSize)
+        } else {
+            // Check that the file name is preserved
+            XCTAssertEqual(originalImageURL.lastPathComponent, convertedImageURL.lastPathComponent)
+            // Check that new image is the same size as the original one
+            XCTAssertEqual(originalImage.size, convertedImage.size)
+        }
        
        // Check that the GPS data has been stripped
        let originalMetadata = metadata(from: originalImageData)
@@ -269,4 +482,15 @@ final class MediaUploadingPreprocessorTests: XCTestCase {
        
        return convertedMetadata
    }
+    
+    private func mimeType(from url: URL) -> String? {
+        guard let imageSource = CGImageSourceCreateWithURL(url as NSURL, nil),
+              let typeIdentifier = CGImageSourceGetType(imageSource),
+              let type = UTType(typeIdentifier as String),
+              let mimeType = type.preferredMIMEType else {
+            XCTFail("Failed to get mimetype from URL.")
+            return nil
+        }
+        return mimeType
+    }
 }