Add voice message rendering to the timeline (#1776)

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/AudioRoomTimelineItemContent.swift

@@ -20,6 +20,7 @@ import UniformTypeIdentifiers
 struct AudioRoomTimelineItemContent: Hashable {
     let body: String
     let duration: TimeInterval
+    let waveform: Waveform?
     let source: MediaSourceProxy?
     let contentType: UTType?
 }

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/VoiceMessages/VoiceRoomPlaybackView.swift

@@ -0,0 +1,167 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import SwiftUI
+
+struct VoiceRoomPlaybackView: View {
+    @ObservedObject var playbackViewState: VoiceRoomPlaybackViewState
+
+    private let feedbackGenerator = UIImpactFeedbackGenerator(style: .heavy)
+    @State private var sendFeedback = false
+    
+    private let waveformMaxWidth: CGFloat = 150
+    private let playPauseButtonSize = CGSize(width: 32, height: 32)
+    
+    private static let elapsedTimeFormatter: DateFormatter = {
+        let dateFormatter = DateFormatter()
+        dateFormatter.dateFormat = "m:ss"
+        return dateFormatter
+    }()
+    
+    var onPlayPause: () -> Void = { }
+    var onSeek: (Double) -> Void = { _ in }
+    var onWaveformDragStateChanged: (Bool) -> Void = { _ in }
+    
+    private enum DragState: Equatable {
+        case inactive
+        case pressing
+        case dragging(progress: Double)
+        
+        var progress: Double {
+            switch self {
+            case .inactive, .pressing:
+                return .zero
+            case .dragging(let progress):
+                return progress
+            }
+        }
+        
+        var isActive: Bool {
+            switch self {
+            case .inactive:
+                return false
+            case .pressing, .dragging:
+                return true
+            }
+        }
+        
+        var isDragging: Bool {
+            switch self {
+            case .inactive, .pressing:
+                return false
+            case .dragging:
+                return true
+            }
+        }
+    }
+    
+    @GestureState private var dragState = DragState.inactive
+    
+    var timeLabelContent: String {
+        // Display the duration if progress is 0.0
+        let percent = playbackViewState.progress > 0.0 ? playbackViewState.progress : 1.0
+        return Self.elapsedTimeFormatter.string(from: Date(timeIntervalSinceReferenceDate: playbackViewState.duration * percent))
+    }
+    
+    var body: some View {
+        HStack {
+            HStack {
+                playPauseButton
+                Text(timeLabelContent)
+                    .font(.compound.bodySMSemibold)
+                    .foregroundColor(.compound.textSecondary)
+                    .monospacedDigit()
+            }
+            .padding(.vertical, 6)
+            GeometryReader { geometry in
+                WaveformView(waveform: playbackViewState.waveform, progress: playbackViewState.progress)
+                    // Add a gesture to drag the waveform
+                    .gesture(LongPressGesture()
+                        .sequenced(before: DragGesture(coordinateSpace: .local))
+                        .updating($dragState) { value, state, _ in
+                            switch value {
+                            // Long press begins.
+                            case .first(true):
+                                state = .pressing
+                            // Long press confirmed, dragging may begin.
+                            case .second(true, let drag):
+                                let progress: Double = (drag?.location.x ?? .zero) / geometry.size.width
+                                state = .dragging(progress: progress)
+                            // Dragging ended or the long press cancelled.
+                            default:
+                                state = .inactive
+                            }
+                        })
+            }
+            .frame(maxWidth: waveformMaxWidth)
+        }
+        .onChange(of: dragState) { newDragState in
+            switch newDragState {
+            case .inactive:
+                onWaveformDragStateChanged(false)
+            case .pressing:
+                onWaveformDragStateChanged(true)
+                feedbackGenerator.prepare()
+                sendFeedback = true
+            case .dragging(let progress):
+                if sendFeedback {
+                    feedbackGenerator.impactOccurred()
+                    sendFeedback = false
+                }
+                if abs(progress - playbackViewState.progress) > 0.01 {
+                    onSeek(max(0, min(progress, 1.0)))
+                }
+            }
+        }
+        .padding(.vertical, 2)
+        .padding(.horizontal, 8)
+    }
+    
+    @ViewBuilder
+    var playPauseButton: some View {
+        Button {
+            onPlayPause()
+        } label: {
+            Image(systemName: playbackViewState.playing ? "pause.fill" : "play.fill")
+                .foregroundColor(.compound.iconSecondary)
+                .background(
+                    Circle()
+                        .frame(width: playPauseButtonSize.width,
+                               height: playPauseButtonSize.height)
+                        .foregroundColor(.compound.bgCanvasDefault)
+                )
+                .padding(.trailing, 7)
+        }
+    }
+}
+
+struct VoiceRoomPlaybackView_Previews: PreviewProvider, TestablePreview {
+    static let waveform = Waveform(data: [3, 127, 400, 266, 126, 122, 373, 251, 45, 112,
+                                          334, 205, 99, 138, 397, 354, 125, 361, 199, 51,
+                                          294, 131, 19, 2, 3, 3, 1, 2, 0, 0,
+                                          0, 0, 0, 0, 0, 3])
+    
+    static let playbackViewState = VoiceRoomPlaybackViewState(duration: 10.0,
+                                                              waveform: waveform,
+                                                              progress: 0.3)
+    
+    static var previews: some View {
+        VoiceRoomPlaybackView(playbackViewState: playbackViewState,
+                              onPlayPause: { playbackViewState.updateState(playing: !playbackViewState.playing) },
+                              onSeek: { playbackViewState.updateState(progress: $0) })
+            .fixedSize(horizontal: false, vertical: true)
+    }
+}

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/VoiceMessages/VoiceRoomPlaybackViewState.swift

@@ -0,0 +1,43 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+import Foundation
+
+@MainActor
+class VoiceRoomPlaybackViewState: ObservableObject {
+    let duration: Double
+    let waveform: Waveform
+    @Published private(set) var loading: Bool
+    @Published private(set) var playing: Bool
+    @Published private(set) var progress: Double
+        
+    init(duration: Double = 0.0, waveform: Waveform? = nil, progress: Double = 0.0) {
+        self.duration = duration
+        self.waveform = waveform ?? Waveform(data: [])
+        self.progress = progress
+        loading = false
+        playing = false
+    }
+    
+    func updateState(progress: Double) {
+        self.progress = max(0.0, min(progress, 1.0))
+    }
+    
+    func updateState(playing: Bool) {
+        self.playing = playing
+    }
+}

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/VoiceMessages/VoiceRoomTimelineView.swift

@@ -0,0 +1,92 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+import SwiftUI
+
+struct VoiceRoomTimelineView: View {
+    @EnvironmentObject private var context: RoomScreenViewModel.Context
+    let timelineItem: VoiceRoomTimelineItem
+    let playbackViewState: VoiceRoomPlaybackViewState
+    
+    init(timelineItem: VoiceRoomTimelineItem, playbackViewState: VoiceRoomPlaybackViewState?) {
+        self.timelineItem = timelineItem
+        if playbackViewState == nil {
+            MXLog.error("[VoiceRoomTimelineView] Voice audio playback state is missing")
+        }
+        self.playbackViewState = playbackViewState ?? VoiceRoomPlaybackViewState()
+    }
+    
+    var body: some View {
+        TimelineStyler(timelineItem: timelineItem) {
+            VoiceRoomPlaybackView(playbackViewState: playbackViewState,
+                                  onPlayPause: onPlaybackPlayPause,
+                                  onSeek: onPlaybackSeek(_:),
+                                  onWaveformDragStateChanged: onPlaybackDragStateChanged(_:))
+                .fixedSize(horizontal: false, vertical: true)
+        }
+    }
+    
+    private func onPlaybackPlayPause() {
+        context.send(viewAction: .playPauseAudio(itemID: timelineItem.id))
+    }
+    
+    private func onPlaybackSeek(_ progress: Double) {
+        context.send(viewAction: .seekAudio(itemID: timelineItem.id, progress: progress))
+    }
+    
+    private func onPlaybackDragStateChanged(_ dragging: Bool) {
+        if dragging {
+            context.send(viewAction: .disableLongPress(itemID: timelineItem.id))
+        } else {
+            context.send(viewAction: .enableLongPress(itemID: timelineItem.id))
+        }
+    }
+}
+
+struct VoiceRoomTimelineView_Previews: PreviewProvider, TestablePreview {
+    static let viewModel = RoomScreenViewModel.mock
+
+    static let voiceRoomTimelineItem = VoiceRoomTimelineItem(id: .random,
+                                                             timestamp: "Now",
+                                                             isOutgoing: false,
+                                                             isEditable: false,
+                                                             isThreaded: false,
+                                                             sender: .init(id: "Bob"),
+                                                             content: .init(body: "audio.ogg",
+                                                                            duration: 300,
+                                                                            waveform: Waveform.mockWaveform,
+                                                                            source: nil,
+                                                                            contentType: nil))
+    
+    static let playbackViewState = VoiceRoomPlaybackViewState(duration: 10.0,
+                                                              waveform: Waveform.mockWaveform,
+                                                              progress: 0.4)
+    
+    static var previews: some View {
+        body.environmentObject(viewModel.context)
+            .previewDisplayName("Bubble")
+        body
+            .environment(\.timelineStyle, .plain)
+            .environmentObject(viewModel.context)
+            .previewDisplayName("Plain")
+    }
+    
+    static var body: some View {
+        VoiceRoomTimelineView(timelineItem: voiceRoomTimelineItem, playbackViewState: playbackViewState)
+            .fixedSize(horizontal: false, vertical: true)
+    }
+}

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/VoiceMessages/WaveformView.swift

@@ -0,0 +1,88 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import SwiftUI
+
+struct Waveform: Equatable, Hashable {
+    let data: [UInt16]
+}
+
+extension Waveform {
+    func normalisedData(count: Int) -> [Float] {
+        guard count > 0 else {
+            return []
+        }
+        let stride = max(1, Int(data.count / count))
+        let data = data.striding(by: stride)
+        let max = data.max().flatMap { Float($0) } ?? 0
+        return data.map { Float($0) / max }
+    }
+}
+
+extension Waveform {
+    static let mockWaveform = Waveform(data: [0, 0, 0, 3, 3, 127, 400, 266, 126, 122, 373, 251, 45, 112,
+                                              334, 205, 99, 138, 397, 354, 125, 361, 199, 51,
+                                              294, 131, 19, 2, 3, 3, 1, 2, 0, 0,
+                                              0, 0])
+}
+
+struct WaveformView: View {
+    let lineWidth: CGFloat = 2
+    let linePadding: CGFloat = 2
+    var waveform: Waveform
+    let minimumGraphAmplitude: CGFloat = 1
+    var progress: CGFloat = 0.0
+    
+    var body: some View {
+        GeometryReader { geometry in
+            ZStack(alignment: .leading) {
+                Rectangle().fill(Color.compound.iconQuaternary)
+                    .frame(width: geometry.size.width, height: geometry.size.height)
+                Rectangle().fill(Color.compound.iconSecondary)
+                    .frame(width: max(0.0, geometry.size.width * progress), height: geometry.size.height)
+            }
+            .mask(alignment: .leading) {
+                Path { path in
+                    let width = geometry.size.width
+                    let height = geometry.size.height
+                    let centerY = geometry.size.height / 2
+                    let visibleSamplesCount = Int(width / (lineWidth + linePadding))
+                    let normalisedData = waveform.normalisedData(count: visibleSamplesCount)
+                    var xOffset: CGFloat = lineWidth / 2
+                    var index = 0
+                    
+                    while xOffset < width - lineWidth {
+                        let sample = CGFloat(index >= normalisedData.count ? 0 : normalisedData[index])
+                        let drawingAmplitude = max(minimumGraphAmplitude, sample * height)
+
+                        path.move(to: CGPoint(x: xOffset, y: centerY - drawingAmplitude / 2))
+                        path.addLine(to: CGPoint(x: xOffset, y: centerY + drawingAmplitude / 2))
+                        xOffset += lineWidth + linePadding
+                        index += 1
+                    }
+                }
+                .stroke(Color.compound.iconSecondary, style: StrokeStyle(lineWidth: lineWidth, lineCap: .round))
+            }
+        }
+    }
+}
+
+struct WaveformView_Previews: PreviewProvider, TestablePreview {
+    static var previews: some View {
+        WaveformView(waveform: Waveform.mockWaveform)
+            .frame(width: 140, height: 50)
+    }
+}

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/VoiceRoomTimelineItem.swift

@@ -0,0 +1,40 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+
+struct VoiceRoomTimelineItem: EventBasedMessageTimelineItemProtocol, Equatable {
+    let id: TimelineItemIdentifier
+    let timestamp: String
+    let isOutgoing: Bool
+    let isEditable: Bool
+    let isThreaded: Bool
+    let sender: TimelineItemSender
+    
+    let content: AudioRoomTimelineItemContent
+    
+    var replyDetails: TimelineItemReplyDetails?
+
+    var properties = RoomTimelineItemProperties()
+    
+    var body: String {
+        content.body
+    }
+    
+    var contentType: EventBasedMessageTimelineItemContentType {
+        .voice(content)
+    }
+}