Audio Timeline Item support (#612)

* audio timeline item support

* improved the code to reduce cognitive complexity

* no need anymore for this swiftlint disabling of a rule

* pr comments

ElementX/Sources/Screens/RoomScreen/View/Timeline/AudioRoomTimelineView.swift

@@ -0,0 +1,55 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+import SwiftUI
+
+struct AudioRoomTimelineView: View {
+    let timelineItem: AudioRoomTimelineItem
+
+    var body: some View {
+        TimelineStyler(timelineItem: timelineItem) {
+            HStack {
+                Image(systemName: "waveform")
+                    .foregroundColor(.element.primaryContent)
+                FormattedBodyText(text: timelineItem.body)
+            }
+            .padding(.vertical, 12)
+            .padding(.horizontal, 6)
+        }
+    }
+}
+
+struct AudioRoomTimelineView_Previews: PreviewProvider {
+    static let viewModel = RoomScreenViewModel.mock
+
+    static var previews: some View {
+        body.environmentObject(viewModel.context)
+        body.timelineStyle(.plain).environmentObject(viewModel.context)
+    }
+
+    static var body: some View {
+        AudioRoomTimelineView(timelineItem: AudioRoomTimelineItem(id: UUID().uuidString,
+                                                                  body: "audio.ogg",
+                                                                  timestamp: "Now",
+                                                                  groupState: .single,
+                                                                  isOutgoing: false,
+                                                                  isEditable: false,
+                                                                  sender: .init(id: "Bob"),
+                                                                  duration: 300,
+                                                                  source: nil))
+    }
+}

ElementX/Sources/Services/Timeline/TimelineController/RoomTimelineController.swift

@@ -101,7 +101,6 @@ class RoomTimelineController: RoomTimelineControllerProtocol {
     
     func processItemDisappearance(_ itemID: String) { }
 
-    // swiftlint:disable:next cyclomatic_complexity
     func processItemTap(_ itemID: String) async -> RoomTimelineControllerAction {
         guard let timelineItem = timelineItems.first(where: { $0.id == itemID }) else {
             return .none
@@ -111,33 +110,36 @@ class RoomTimelineController: RoomTimelineControllerProtocol {
         case let item as ImageRoomTimelineItem:
             await loadFileForImageTimelineItem(item)
             guard let index = timelineItems.firstIndex(where: { $0.id == itemID }),
-                  let item = timelineItems[index] as? ImageRoomTimelineItem else {
+                  let item = timelineItems[index] as? ImageRoomTimelineItem,
+                  let fileURL = item.cachedFileURL else {
                 return .none
             }
-            if let fileURL = item.cachedFileURL {
-                return .displayFile(fileURL: fileURL, title: item.body)
-            }
-            return .none
+            return .displayFile(fileURL: fileURL, title: item.body)
         case let item as VideoRoomTimelineItem:
             await loadVideoForTimelineItem(item)
             guard let index = timelineItems.firstIndex(where: { $0.id == itemID }),
-                  let item = timelineItems[index] as? VideoRoomTimelineItem else {
+                  let item = timelineItems[index] as? VideoRoomTimelineItem,
+                  let videoURL = item.cachedVideoURL else {
                 return .none
             }
-            if let videoURL = item.cachedVideoURL {
-                return .displayVideo(videoURL: videoURL, title: item.body)
-            }
-            return .none
+            return .displayVideo(videoURL: videoURL, title: item.body)
         case let item as FileRoomTimelineItem:
             await loadFileForTimelineItem(item)
             guard let index = timelineItems.firstIndex(where: { $0.id == itemID }),
-                  let item = timelineItems[index] as? FileRoomTimelineItem else {
+                  let item = timelineItems[index] as? FileRoomTimelineItem,
+                  let fileURL = item.cachedFileURL else {
                 return .none
             }
-            if let fileURL = item.cachedFileURL {
-                return .displayFile(fileURL: fileURL, title: item.body)
+            return .displayFile(fileURL: fileURL, title: item.body)
+        case let item as AudioRoomTimelineItem:
+            await loadAudioForTimelineItem(item)
+            guard let index = timelineItems.firstIndex(where: { $0.id == itemID }),
+                  let item = timelineItems[index] as? AudioRoomTimelineItem,
+                  let audioURL = item.cachedAudioURL else {
+                return .none
             }
-            return .none
+            // For now we are just displaying audio messages with the File preview until we create a timeline player for them.
+            return .displayFile(fileURL: audioURL, title: item.body)
         default:
             return .none
         }
@@ -437,4 +439,33 @@ class RoomTimelineController: RoomTimelineControllerProtocol {
             break
         }
     }
+
+    private func loadAudioForTimelineItem(_ timelineItem: AudioRoomTimelineItem) async {
+        if timelineItem.cachedAudioURL != nil {
+            // already cached
+            return
+        }
+
+        guard let source = timelineItem.source else {
+            return
+        }
+
+        // This is not great. We could better estimate file extension from the mimetype.
+        guard let fileExtension = timelineItem.body.split(separator: ".").last else {
+            return
+        }
+
+        switch await mediaProvider.loadFileFromSource(source, fileExtension: String(fileExtension)) {
+        case .success(let audioURL):
+            guard let index = timelineItems.firstIndex(where: { $0.id == timelineItem.id }),
+                  var item = timelineItems[index] as? AudioRoomTimelineItem else {
+                return
+            }
+
+            item.cachedAudioURL = audioURL
+            timelineItems[index] = item
+        case .failure:
+            break
+        }
+    }
 }

ElementX/Sources/Services/Timeline/TimelineItemContent/CustomStringConvertible.swift

@@ -53,3 +53,9 @@ extension VideoMessageContent: CustomStringConvertible {
         String(describing: Self.self)
     }
 }
+
+extension AudioMessageContent: CustomStringConvertible {
+    public var description: String {
+        String(describing: Self.self)
+    }
+}

ElementX/Sources/Services/Timeline/TimelineItemContent/MessageTimelineItem.swift

@@ -149,3 +149,16 @@ extension MessageTimelineItem where Content == MatrixRustSDK.FileMessageContent
         return .init(source: src)
     }
 }
+
+extension MatrixRustSDK.AudioMessageContent: MessageContentProtocol { }
+
+/// A timeline item that represents an `m.room.message` event with a `msgtype` of `m.audio`.
+extension MessageTimelineItem where Content == MatrixRustSDK.AudioMessageContent {
+    var source: MediaSourceProxy {
+        .init(source: content.source)
+    }
+
+    var duration: UInt64 {
+        content.info?.duration ?? 0
+    }
+}

ElementX/Sources/Services/Timeline/TimelineItems/EventBasedTimelineItemProtocol.swift

@@ -33,7 +33,7 @@ enum TimelineItemGroupState: Hashable {
     }
 }
 
-protocol EventBasedTimelineItemProtocol: RoomTimelineItemProtocol {
+protocol EventBasedTimelineItemProtocol: RoomTimelineItemProtocol, CustomStringConvertible {
     var body: String { get }
     var timestamp: String { get }
     var shouldShowSenderDetails: Bool { get }
@@ -71,4 +71,8 @@ extension EventBasedTimelineItemProtocol {
             }
         }
     }
+
+    var description: String {
+        "\(String(describing: Self.self)): id: \(id), timestamp: \(timestamp), isOutgoing: \(isOutgoing), properties: \(properties)"
+    }
 }

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/AudioRoomTimelineItem.swift

@@ -0,0 +1,33 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+
+struct AudioRoomTimelineItem: EventBasedTimelineItemProtocol, Identifiable, Hashable {
+    let id: String
+    let body: String
+    let timestamp: String
+    let groupState: TimelineItemGroupState
+    let isOutgoing: Bool
+    let isEditable: Bool
+    let sender: TimelineItemSender
+
+    let duration: UInt64
+    let source: MediaSourceProxy?
+    var cachedAudioURL: URL?
+
+    var properties = RoomTimelineItemProperties()
+}

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/EmoteRoomTimelineItem.swift

@@ -29,11 +29,3 @@ struct EmoteRoomTimelineItem: EventBasedTimelineItemProtocol, Identifiable, Hash
     
     var properties = RoomTimelineItemProperties()
 }
-
-// MARK: - Redact content from logs
-
-extension EmoteRoomTimelineItem: CustomStringConvertible {
-    var description: String {
-        "\(String(describing: Self.self)): id: \(id), timestamp: \(timestamp), isOutgoing: \(isOutgoing), properties: \(properties)"
-    }
-}

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/FileRoomTimelineItem.swift

@@ -32,11 +32,3 @@ struct FileRoomTimelineItem: EventBasedTimelineItemProtocol, Identifiable, Hasha
     
     var properties = RoomTimelineItemProperties()
 }
-
-// MARK: - Redact content from logs
-
-extension FileRoomTimelineItem: CustomStringConvertible {
-    var description: String {
-        "\(String(describing: Self.self)): id: \(id), timestamp: \(timestamp), isOutgoing: \(isOutgoing), properties: \(properties)"
-    }
-}

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/ImageRoomTimelineItem.swift

@@ -38,11 +38,3 @@ struct ImageRoomTimelineItem: EventBasedTimelineItemProtocol, Identifiable, Hash
     
     var properties = RoomTimelineItemProperties()
 }
-
-// MARK: - Redact content from logs
-
-extension ImageRoomTimelineItem: CustomStringConvertible {
-    var description: String {
-        "\(String(describing: Self.self)): id: \(id), timestamp: \(timestamp), isOutgoing: \(isOutgoing), properties: \(properties)"
-    }
-}

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/NoticeRoomTimelineItem.swift

@@ -29,11 +29,3 @@ struct NoticeRoomTimelineItem: EventBasedTimelineItemProtocol, Identifiable, Has
     
     var properties = RoomTimelineItemProperties()
 }
-
-// MARK: - Redact content from logs
-
-extension NoticeRoomTimelineItem: CustomStringConvertible {
-    var description: String {
-        "\(String(describing: Self.self)): id: \(id), timestamp: \(timestamp), isOutgoing: \(isOutgoing), properties: \(properties)"
-    }
-}

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/TextRoomTimelineItem.swift

@@ -29,11 +29,3 @@ struct TextRoomTimelineItem: EventBasedTimelineItemProtocol, Identifiable, Hasha
     
     var properties = RoomTimelineItemProperties()
 }
-
-// MARK: - Redact content from logs
-
-extension TextRoomTimelineItem: CustomStringConvertible {
-    var description: String {
-        "\(String(describing: Self.self)): id: \(id), timestamp: \(timestamp), isOutgoing: \(isOutgoing), properties: \(properties)"
-    }
-}

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/VideoRoomTimelineItem.swift

@@ -38,11 +38,3 @@ struct VideoRoomTimelineItem: EventBasedTimelineItemProtocol, Identifiable, Hash
     
     var properties = RoomTimelineItemProperties()
 }
-
-// MARK: - Redact content from logs
-
-extension VideoRoomTimelineItem: CustomStringConvertible {
-    var description: String {
-        "\(String(describing: Self.self)): id: \(id), timestamp: \(timestamp), isOutgoing: \(isOutgoing), properties: \(properties)"
-    }
-}

ElementX/Sources/Services/Timeline/TimelineItems/RoomTimelineItemFactory.swift

@@ -72,15 +72,15 @@ struct RoomTimelineItemFactory: RoomTimelineItemFactoryProtocol {
             case .file(let content):
                 let message = MessageTimelineItem(item: eventItemProxy.item, content: content)
                 return buildFileTimelineItemFromMessage(eventItemProxy, message, isOutgoing, groupState)
-            case .notice(content: let content):
+            case .notice(let content):
                 let message = MessageTimelineItem(item: eventItemProxy.item, content: content)
                 return buildNoticeTimelineItemFromMessage(eventItemProxy, message, isOutgoing, groupState)
-            case .emote(content: let content):
+            case .emote(let content):
                 let message = MessageTimelineItem(item: eventItemProxy.item, content: content)
                 return buildEmoteTimelineItemFromMessage(eventItemProxy, message, isOutgoing, groupState)
-            case .audio:
-                #warning("Audio timeline items not currently supported")
-                return nil
+            case .audio(let content):
+                let message = MessageTimelineItem(item: eventItemProxy.item, content: content)
+                return buildAudioTimelineItem(eventItemProxy, message, isOutgoing, groupState)
             case .none:
                 return buildFallbackTimelineItem(eventItemProxy, isOutgoing, groupState)
             }
@@ -273,6 +273,21 @@ struct RoomTimelineItemFactory: RoomTimelineItemFactoryProtocol {
                                                                             deliveryStatus: eventItemProxy.deliveryStatus))
     }
 
+    private func buildAudioTimelineItem(_ eventItemProxy: EventTimelineItemProxy,
+                                        _ message: MessageTimelineItem<AudioMessageContent>,
+                                        _ isOutgoing: Bool,
+                                        _ groupState: TimelineItemGroupState) -> RoomTimelineItemProtocol {
+        AudioRoomTimelineItem(id: eventItemProxy.id,
+                              body: message.body,
+                              timestamp: message.timestamp.formatted(date: .omitted, time: .shortened),
+                              groupState: groupState,
+                              isOutgoing: isOutgoing,
+                              isEditable: eventItemProxy.isEditable,
+                              sender: eventItemProxy.sender,
+                              duration: message.duration,
+                              source: message.source)
+    }
+
     private func buildFileTimelineItemFromMessage(_ eventItemProxy: EventTimelineItemProxy,
                                                   _ message: MessageTimelineItem<FileMessageContent>,
                                                   _ isOutgoing: Bool,

ElementX/Sources/Services/Timeline/TimelineItems/RoomTimelineViewFactory.swift

@@ -26,6 +26,8 @@ struct RoomTimelineViewFactory: RoomTimelineViewFactoryProtocol {
             return .image(item)
         case let item as VideoRoomTimelineItem:
             return .video(item)
+        case let item as AudioRoomTimelineItem:
+            return .audio(item)
         case let item as FileRoomTimelineItem:
             return .file(item)
         case let item as SeparatorRoomTimelineItem:

ElementX/Sources/Services/Timeline/TimelineItems/RoomTimelineViewProvider.swift

@@ -22,6 +22,7 @@ enum RoomTimelineViewProvider: Identifiable, Hashable {
     case separator(SeparatorRoomTimelineItem)
     case image(ImageRoomTimelineItem)
     case video(VideoRoomTimelineItem)
+    case audio(AudioRoomTimelineItem)
     case file(FileRoomTimelineItem)
     case emote(EmoteRoomTimelineItem)
     case notice(NoticeRoomTimelineItem)
@@ -44,6 +45,8 @@ enum RoomTimelineViewProvider: Identifiable, Hashable {
             return item.id
         case .video(let item):
             return item.id
+        case .audio(let item):
+            return item.id
         case .file(let item):
             return item.id
         case .emote(let item):
@@ -72,7 +75,7 @@ enum RoomTimelineViewProvider: Identifiable, Hashable {
     /// Whether or not it is possible to send a reaction to this timeline item.
     var isReactable: Bool {
         switch self {
-        case .text, .image, .video, .file, .emote, .notice, .sticker:
+        case .text, .image, .video, .audio, .file, .emote, .notice, .sticker:
             return true
         case .redacted, .encrypted, .unsupported, .state: // Event based items that aren't reactable
             return false
@@ -93,6 +96,8 @@ extension RoomTimelineViewProvider: View {
             ImageRoomTimelineView(timelineItem: item)
         case .video(let item):
             VideoRoomTimelineView(timelineItem: item)
+        case .audio(let item):
+            AudioRoomTimelineView(timelineItem: item)
         case .file(let item):
             FileRoomTimelineView(timelineItem: item)
         case .emote(let item):

NSE/Sources/Other/NotificationItemProxy+NSE.swift

@@ -117,6 +117,11 @@ extension NotificationItemProxy {
                                           senderId: senderId,
                                           roomId: roomId,
                                           mediaProvider: mediaProvider)
+        case .audio(content: let content):
+            return try await processAudio(content: content,
+                                          senderId: senderId,
+                                          roomId: roomId,
+                                          mediaProvider: mediaProvider)
         case .none:
             return nil
         }
@@ -222,4 +227,16 @@ extension NotificationItemProxy {
 
         return notification
     }
+
+    private func processAudio(content: AudioMessageContent,
+                              senderId: String,
+                              roomId: String,
+                              mediaProvider: MediaProviderProtocol?) async throws -> UNMutableNotificationContent {
+        let notification = try await processCommon(senderId: senderId,
+                                                   roomId: roomId,
+                                                   mediaProvider: mediaProvider)
+        notification.body = "🔊 " + content.body
+
+        return notification
+    }
 }

changelog.d/594.feature

@@ -0,0 +1 @@
+Added support for audio messages in the timeline as previewable files.