Voice message waveform from audio files (#1919)

* Add DSWaveformImage * Add estimated waveform * Add progress mask modifier * Add real waveform * Fix cursor * Fix scale factor * Refine ProgressMaskModifier * Refine waveform logic * Add docs * Add waveform placeholder
2026-11-04 19:44:05 +00:00 · 2023-10-19 11:01:45 +02:00
parent 75da7ba3ec
commit fbcf037240
16 changed files with 175 additions and 61 deletions
@@ -51,7 +51,7 @@ struct VoiceMessagePreviewComposer: View {

 struct VoiceMessagePreviewComposer_Previews: PreviewProvider, TestablePreview {
    static let playerState = AudioPlayerState(duration: 10.0,
-                                              waveform: Waveform.mockWaveform,
+                                              waveform: EstimatedWaveform.mockWaveform,
                                              progress: 0.4)
    
    static var previews: some View {
@@ -45,7 +45,7 @@ struct VoiceMessageRecordingView: View {
                .foregroundColor(.compound.textSecondary)
                .monospacedDigit()
                .fixedSize()
-            WaveformView(lineWidth: waveformLineWidth, linePadding: waveformLinePadding, waveform: recorderState.waveform, progress: 0, showCursor: false)
+            EstimatedWaveformView(lineWidth: waveformLineWidth, linePadding: waveformLinePadding, waveform: recorderState.waveform, progress: 0)
        }
        .padding(.leading, 2)
        .padding(.trailing, 8)
@@ -53,10 +53,10 @@ struct VoiceMessageRecordingView: View {
 }

 struct VoiceMessageRecordingView_Previews: PreviewProvider, TestablePreview {
-    static let waveform = Waveform(data: [3, 127, 400, 266, 126, 122, 373, 251, 45, 112,
-                                          334, 205, 99, 138, 397, 354, 125, 361, 199, 51,
-                                          294, 131, 19, 2, 3, 3, 1, 2, 0, 0,
-                                          0, 0, 0, 0, 0, 3])
+    static let waveform = EstimatedWaveform(data: [3, 127, 400, 266, 126, 122, 373, 251, 45, 112,
+                                                   334, 205, 99, 138, 397, 354, 125, 361, 199, 51,
+                                                   294, 131, 19, 2, 3, 3, 1, 2, 0, 0,
+                                                   0, 0, 0, 0, 0, 3])
    
    static let recorderState = AudioRecorderState()
    
@@ -437,7 +437,7 @@ struct TimelineItemBubbledStylerView_Previews: PreviewProvider, TestablePreview
                                                      sender: .init(id: ""),
                                                      content: .init(body: "audio.ogg",
                                                                     duration: 100,
-                                                                     waveform: Waveform.mockWaveform,
+                                                                     waveform: EstimatedWaveform.mockWaveform,
                                                                     source: nil,
                                                                     contentType: nil),
                                                      replyDetails: .loaded(sender: .init(id: "", displayName: "Alice"),
@@ -500,12 +500,12 @@ struct TimelineItemBubbledStylerView_Previews: PreviewProvider, TestablePreview
                                                             sender: .init(id: ""),
                                                             content: .init(body: "audio.ogg",
                                                                            duration: 100,
-                                                                            waveform: Waveform.mockWaveform,
+                                                                            waveform: EstimatedWaveform.mockWaveform,
                                                                            source: nil,
                                                                            contentType: nil),
                                                             replyDetails: .loaded(sender: .init(id: "", displayName: "Alice"),
                                                                                   contentType: .text(.init(body: "Short")))),
-                                         playerState: AudioPlayerState(duration: 10, waveform: Waveform.mockWaveform))
+                                         playerState: AudioPlayerState(duration: 10, waveform: EstimatedWaveform.mockWaveform))
        }
        .environmentObject(viewModel.context)
    }
@@ -227,12 +227,12 @@ struct TimelineItemPlainStylerView_Previews: PreviewProvider, TestablePreview {
                                                             sender: .init(id: ""),
                                                             content: .init(body: "audio.ogg",
                                                                            duration: 100,
-                                                                            waveform: Waveform.mockWaveform,
+                                                                            waveform: EstimatedWaveform.mockWaveform,
                                                                            source: nil,
                                                                            contentType: nil),
                                                             replyDetails: .loaded(sender: .init(id: "", displayName: "Alice"),
                                                                                   contentType: .text(.init(body: "Short")))),
-                                         playerState: AudioPlayerState(duration: 10, waveform: Waveform.mockWaveform))
+                                         playerState: AudioPlayerState(duration: 10, waveform: EstimatedWaveform.mockWaveform))
        }
        .environmentObject(viewModel.context)
    }
@@ -30,7 +30,7 @@ enum AudioPlayerPlaybackState {
 class AudioPlayerState: ObservableObject, Identifiable {
    let id = UUID()
    let duration: Double
-    let waveform: Waveform
+    let waveform: EstimatedWaveform
    @Published private(set) var playbackState: AudioPlayerPlaybackState
    @Published private(set) var progress: Double

@@ -38,6 +38,10 @@ class AudioPlayerState: ObservableObject, Identifiable {
    private var cancellables: Set<AnyCancellable> = []
    private var displayLink: CADisplayLink?

+    /// The file url that the last player attached to this object has loaded.
+    /// The file url persists even if the AudioPlayer will be detached later.
+    private(set) var fileURL: URL?
+
    var isAttached: Bool {
        audioPlayer != nil
    }
@@ -46,9 +50,9 @@ class AudioPlayerState: ObservableObject, Identifiable {
        displayLink != nil
    }

-    init(duration: Double, waveform: Waveform? = nil, progress: Double = 0.0) {
+    init(duration: Double, waveform: EstimatedWaveform? = nil, progress: Double = 0.0) {
        self.duration = duration
-        self.waveform = waveform ?? Waveform(data: [])
+        self.waveform = waveform ?? EstimatedWaveform(data: [])
        self.progress = progress
        playbackState = .stopped
    }
@@ -110,6 +114,7 @@ class AudioPlayerState: ObservableObject, Identifiable {
            playbackState = .loading
        case .didFinishLoading:
            playbackState = .readyToPlay
+            fileURL = audioPlayer?.url
        case .didStartPlaying:
            if let audioPlayer {
                await restoreAudioPlayerState(audioPlayer: audioPlayer)
@@ -30,7 +30,7 @@ class AudioRecorderState: ObservableObject, Identifiable {
    
    @Published private(set) var recordingState: AudioRecorderRecordingState = .stopped
    @Published private(set) var duration = 0.0
-    @Published private(set) var waveform = Waveform(data: Array(repeating: 0, count: 100))
+    @Published private(set) var waveform = EstimatedWaveform(data: Array(repeating: 0, count: 100))
    
    private weak var audioRecorder: AudioRecorderProtocol?
    private var cancellables: Set<AnyCancellable> = []
@@ -20,7 +20,7 @@ import UniformTypeIdentifiers
 struct AudioRoomTimelineItemContent: Hashable {
    let body: String
    let duration: TimeInterval
-    let waveform: Waveform?
+    let waveform: EstimatedWaveform?
    let source: MediaSourceProxy?
    let contentType: UTType?
 }
@@ -16,11 +16,11 @@

 import SwiftUI

-struct Waveform: Equatable, Hashable {
+struct EstimatedWaveform: Equatable, Hashable {
    let data: [UInt16]
 }

-extension Waveform {
+extension EstimatedWaveform {
    func normalisedData(keepSamplesCount: Int) -> [Float] {
        guard keepSamplesCount > 0 else {
            return []
@@ -43,45 +43,30 @@ extension Waveform {
    }
 }

-extension Waveform {
-    static let mockWaveform = Waveform(data: [0, 0, 0, 3, 3, 127, 400, 266, 126, 122, 373, 251, 45, 112,
-                                              334, 205, 99, 138, 397, 354, 125, 361, 199, 51,
-                                              294, 131, 19, 2, 3, 3, 1, 2, 0, 0,
-                                              0, 0])
+extension EstimatedWaveform {
+    static let mockWaveform = EstimatedWaveform(data: [0, 0, 0, 3, 3, 127, 400, 266, 126, 122, 373, 251, 45, 112,
+                                                       334, 205, 99, 138, 397, 354, 125, 361, 199, 51,
+                                                       294, 131, 19, 2, 3, 3, 1, 2, 0, 0,
+                                                       0, 0])
 }

-struct WaveformView: View {
+struct EstimatedWaveformView: View {
    var lineWidth: CGFloat = 2
    var linePadding: CGFloat = 2
-    var waveform: Waveform
+    var waveform: EstimatedWaveform
    private let minimumGraphAmplitude: CGFloat = 1
    var progress: CGFloat = 0.0
-    var showCursor = false
    
    @State private var normalizedWaveformData: [Float] = []
    
    var body: some View {
        GeometryReader { geometry in
-            ZStack(alignment: .leading) {
-                Rectangle().fill(Color.compound.iconQuaternary)
-                    .frame(width: geometry.size.width, height: geometry.size.height)
-                Rectangle().fill(Color.compound.iconSecondary)
-                    .frame(width: max(0.0, geometry.size.width * progress), height: geometry.size.height)
-            }
-            .preference(key: ViewSizeKey.self, value: geometry.size)
-            .mask(alignment: .leading) {
-                WaveformShape(lineWidth: lineWidth,
-                              linePadding: linePadding,
-                              waveformData: normalizedWaveformData)
-                    .stroke(Color.compound.iconSecondary, style: StrokeStyle(lineWidth: lineWidth, lineCap: .round))
-            }
-            // Display a cursor
-            .overlay(alignment: .leading) {
-                RoundedRectangle(cornerRadius: 1).fill(Color.compound.iconAccentTertiary)
-                    .offset(CGSize(width: progress * geometry.size.width, height: 0.0))
-                    .frame(width: lineWidth, height: geometry.size.height)
-                    .opacity(showCursor ? 1 : 0)
-            }
+            WaveformShape(lineWidth: lineWidth,
+                          linePadding: linePadding,
+                          waveformData: normalizedWaveformData)
+                .stroke(Color.compound.iconSecondary, style: StrokeStyle(lineWidth: lineWidth, lineCap: .round))
+                .progressMask(progress: progress)
+                .preference(key: ViewSizeKey.self, value: geometry.size)
        }
        .onPreferenceChange(ViewSizeKey.self) { size in
            buildNormalizedWaveformData(size: size)
@@ -122,7 +107,7 @@ private struct WaveformShape: Shape {
        while xOffset <= width {
            let sample = CGFloat(index >= waveformData.count ? 0 : waveformData[index])
            let drawingAmplitude = max(minimumGraphAmplitude, sample * (height - 2))
-
+            
            path.move(to: CGPoint(x: xOffset, y: centerY - drawingAmplitude / 2))
            path.addLine(to: CGPoint(x: xOffset, y: centerY + drawingAmplitude / 2))
            xOffset += lineWidth + linePadding
@@ -137,7 +122,7 @@ struct WaveformView_Previews: PreviewProvider, TestablePreview {
    static var previews: some View {
        // Wrap the WaveformView in a VStack otherwise the preview test will fail (because of Prefire / GeometryReader)
        VStack {
-            WaveformView(waveform: Waveform.mockWaveform, progress: 0.5)
+            EstimatedWaveformView(waveform: EstimatedWaveform.mockWaveform, progress: 0.5)
                .frame(width: 140, height: 50)
        }
    }
@@ -0,0 +1,56 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import SwiftUI
+
+extension View {
+    func progressMask(progress: CGFloat,
+                      trackColor: Color = .compound.iconSecondary,
+                      backgroundTrackColor: Color = .compound.iconQuaternary) -> some View {
+        modifier(ProgressMaskModifier(progress: progress,
+                                      trackColor: trackColor,
+                                      backgroundTrackColor: backgroundTrackColor))
+    }
+}
+
+private struct ProgressMaskModifier: ViewModifier {
+    private let progress: CGFloat
+    private let trackColor: Color
+    private let backgroundTrackColor: Color
+
+    init(progress: CGFloat, trackColor: Color, backgroundTrackColor: Color) {
+        self.progress = progress
+        self.trackColor = trackColor
+        self.backgroundTrackColor = backgroundTrackColor
+    }
+
+    func body(content: Content) -> some View {
+        GeometryReader { geometry in
+            ZStack(alignment: .leading) {
+                Rectangle()
+                    .fill(backgroundTrackColor)
+                    .frame(width: geometry.size.width, height: geometry.size.height)
+                
+                Rectangle()
+                    .fill(trackColor)
+                    .frame(width: max(0.0, geometry.size.width * progress), height: geometry.size.height)
+            }
+            .mask {
+                content
+            }
+        }
+    }
+}
@@ -14,6 +14,8 @@
 // limitations under the License.
 //

+import DSWaveformImage
+import DSWaveformImageViews
 import SwiftUI

 struct VoiceMessageRoomPlaybackView: View {
@@ -74,7 +76,15 @@ struct VoiceMessageRoomPlaybackView: View {
                    .fixedSize(horizontal: true, vertical: true)
            }
            GeometryReader { geometry in
-                WaveformView(lineWidth: waveformLineWidth, linePadding: waveformLinePadding, waveform: playerState.waveform, progress: playerState.progress, showCursor: showWaveformCursor)
+                waveformView
+                    .overlay(alignment: .leading) {
+                        // Display a cursor
+                        RoundedRectangle(cornerRadius: 1)
+                            .fill(Color.compound.iconAccentTertiary)
+                            .offset(CGSize(width: playerState.progress * geometry.size.width, height: 0.0))
+                            .frame(width: waveformLineWidth, height: geometry.size.height)
+                            .opacity(showWaveformCursor ? 1 : 0)
+                    }
                    // Add a gesture to drag the waveform
                    .gesture(SpatialTapGesture()
                        .simultaneously(with: LongPressGesture())
@@ -147,6 +157,26 @@ struct VoiceMessageRoomPlaybackView: View {
        .frame(width: playPauseButtonSize,
               height: playPauseButtonSize)
    }
+
+    @ViewBuilder
+    private var waveformView: some View {
+        if let url = playerState.fileURL {
+            WaveformView(audioURL: url,
+                         configuration: .init(style: .striped(.init(color: .black, width: waveformLineWidth, spacing: waveformLinePadding)),
+                                              verticalScalingFactor: 1.0),
+                         placeholder: { estimatedWaveformView })
+                .progressMask(progress: playerState.progress)
+        } else {
+            estimatedWaveformView
+        }
+    }
+
+    private var estimatedWaveformView: some View {
+        EstimatedWaveformView(lineWidth: waveformLineWidth,
+                              linePadding: waveformLinePadding,
+                              waveform: playerState.waveform,
+                              progress: playerState.progress)
+    }
 }

 private enum DragState: Equatable {
@@ -183,10 +213,10 @@ private enum DragState: Equatable {
 }

 struct VoiceMessageRoomPlaybackView_Previews: PreviewProvider, TestablePreview {
-    static let waveform = Waveform(data: [3, 127, 400, 266, 126, 122, 373, 251, 45, 112,
-                                          334, 205, 99, 138, 397, 354, 125, 361, 199, 51,
-                                          294, 131, 19, 2, 3, 3, 1, 2, 0, 0,
-                                          0, 0, 0, 0, 0, 3])
+    static let waveform = EstimatedWaveform(data: [3, 127, 400, 266, 126, 122, 373, 251, 45, 112,
+                                                   334, 205, 99, 138, 397, 354, 125, 361, 199, 51,
+                                                   294, 131, 19, 2, 3, 3, 1, 2, 0, 0,
+                                                   0, 0, 0, 0, 0, 3])
    
    static var playerState = AudioPlayerState(duration: 10.0,
                                              waveform: waveform,
@@ -75,12 +75,12 @@ struct VoiceMessageRoomTimelineView_Previews: PreviewProvider, TestablePreview {
                                                                    sender: .init(id: "Bob"),
                                                                    content: .init(body: "audio.ogg",
                                                                                   duration: 300,
-                                                                                   waveform: Waveform.mockWaveform,
+                                                                                   waveform: EstimatedWaveform.mockWaveform,
                                                                                   source: nil,
                                                                                   contentType: nil))
    
    static let playerState = AudioPlayerState(duration: 10.0,
-                                              waveform: Waveform.mockWaveform,
+                                              waveform: EstimatedWaveform.mockWaveform,
                                              progress: 0.4)
    
    static var previews: some View {
@@ -469,9 +469,9 @@ struct RoomTimelineItemFactory: RoomTimelineItemFactoryProtocol {
    }
    
    private func buildAudioTimelineItemContent(_ messageContent: AudioMessageContent) -> AudioRoomTimelineItemContent {
-        var waveform: Waveform?
+        var waveform: EstimatedWaveform?
        if let audioWaveform = messageContent.audio?.waveform {
-            waveform = Waveform(data: audioWaveform)
+            waveform = EstimatedWaveform(data: audioWaveform)
        }

        return AudioRoomTimelineItemContent(body: messageContent.body,