Add UI to record a voice message from the composer toolbar (#1892)

ElementX/Sources/Services/Audio/AudioConverter.swift

@@ -30,6 +30,7 @@ enum AudioConverterPreferredFileExtension: String {
 struct AudioConverter: AudioConverterProtocol {
     func convertToOpusOgg(sourceURL: URL, destinationURL: URL) throws {
         do {
+            MXLog.debug("converting \(sourceURL) to \(destinationURL)")
             try OGGConverter.convertM4aFileToOpusOGG(src: sourceURL, dest: destinationURL)
         } catch {
             MXLog.error("failed to convert to OpusOgg: \(error)")

ElementX/Sources/Services/Audio/Player/AudioPlayerState.swift

@@ -27,7 +27,8 @@ enum AudioPlayerPlaybackState {
 }
 
 @MainActor
-class AudioPlayerState: ObservableObject {
+class AudioPlayerState: ObservableObject, Identifiable {
+    let id = UUID()
     let duration: Double
     let waveform: Waveform
     @Published private(set) var playbackState: AudioPlayerPlaybackState
@@ -150,3 +151,9 @@ class AudioPlayerState: ObservableObject {
         await audioPlayer.seek(to: progress)
     }
 }
+
+extension AudioPlayerState: Equatable {
+    nonisolated static func == (lhs: AudioPlayerState, rhs: AudioPlayerState) -> Bool {
+        lhs.id == rhs.id
+    }
+}

ElementX/Sources/Services/Audio/Recorder/AudioRecorder.swift

@@ -0,0 +1,115 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import AVFoundation
+import Combine
+import Foundation
+
+enum AudioRecorderError: Error {
+    case genericError
+}
+
+class AudioRecorder: NSObject, AudioRecorderProtocol, AVAudioRecorderDelegate {
+    private let silenceThreshold: Float = -50.0
+    
+    private var audioRecorder: AVAudioRecorder?
+    
+    private let actionsSubject: PassthroughSubject<AudioRecorderAction, Never> = .init()
+    var actions: AnyPublisher<AudioRecorderAction, Never> {
+        actionsSubject.eraseToAnyPublisher()
+    }
+    
+    var url: URL? {
+        audioRecorder?.url
+    }
+    
+    var currentTime: TimeInterval {
+        audioRecorder?.currentTime ?? 0
+    }
+    
+    var isRecording: Bool {
+        audioRecorder?.isRecording ?? false
+    }
+    
+    func recordWithOutputURL(_ url: URL) {
+        let settings = [AVFormatIDKey: Int(kAudioFormatMPEG4AAC),
+                        AVSampleRateKey: 48000,
+                        AVEncoderBitRateKey: 128_000,
+                        AVNumberOfChannelsKey: 1,
+                        AVEncoderAudioQualityKey: AVAudioQuality.high.rawValue]
+        
+        do {
+            try AVAudioSession.sharedInstance().setCategory(.playAndRecord, mode: .default)
+            try AVAudioSession.sharedInstance().setActive(true)
+            audioRecorder = try AVAudioRecorder(url: url, settings: settings)
+            audioRecorder?.delegate = self
+            audioRecorder?.isMeteringEnabled = true
+            audioRecorder?.record()
+            actionsSubject.send(.didStartRecording)
+        } catch {
+            MXLog.error("audio recording failed: \(error)")
+            actionsSubject.send(.didFailWithError(error: error))
+        }
+    }
+
+    func stopRecording() {
+        audioRecorder?.stop()
+
+        do {
+            try AVAudioSession.sharedInstance().setActive(false)
+        } catch {
+            actionsSubject.send(.didFailWithError(error: error))
+        }
+    }
+    
+    func peakPowerForChannelNumber(_ channelNumber: Int) -> Float {
+        guard isRecording, let audioRecorder else {
+            return 0.0
+        }
+        
+        audioRecorder.updateMeters()
+
+        return normalizedPowerLevelFromDecibels(audioRecorder.peakPower(forChannel: channelNumber))
+    }
+    
+    func averagePowerForChannelNumber(_ channelNumber: Int) -> Float {
+        guard isRecording, let audioRecorder else {
+            return 0.0
+        }
+        
+        audioRecorder.updateMeters()
+        
+        return normalizedPowerLevelFromDecibels(audioRecorder.averagePower(forChannel: channelNumber))
+    }
+        
+    // MARK: - AVAudioRecorderDelegate
+    
+    func audioRecorderDidFinishRecording(_ recorder: AVAudioRecorder, successfully success: Bool) {
+        if success {
+            actionsSubject.send(.didStopRecording)
+        } else {
+            actionsSubject.send(.didFailWithError(error: AudioRecorderError.genericError))
+        }
+    }
+    
+    func audioRecorderEncodeErrorDidOccur(_ recorder: AVAudioRecorder, error: Error?) {
+        actionsSubject.send(.didFailWithError(error: error ?? AudioRecorderError.genericError))
+    }
+    
+    private func normalizedPowerLevelFromDecibels(_ decibels: Float) -> Float {
+        decibels / silenceThreshold
+    }
+}

ElementX/Sources/Services/Audio/Recorder/AudioRecorderProtocol.swift

@@ -0,0 +1,38 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+import Foundation
+
+enum AudioRecorderAction {
+    case didStartRecording
+    case didStopRecording
+    case didFailWithError(error: Error)
+}
+
+protocol AudioRecorderProtocol: AnyObject {
+    var actions: AnyPublisher<AudioRecorderAction, Never> { get }
+    var currentTime: TimeInterval { get }
+    var isRecording: Bool { get }
+    var url: URL? { get }
+    
+    func recordWithOutputURL(_ url: URL)
+    func stopRecording()
+    func averagePowerForChannelNumber(_ channelNumber: Int) -> Float
+}
+
+// sourcery: AutoMockable
+extension AudioRecorderProtocol { }

ElementX/Sources/Services/Audio/Recorder/AudioRecorderState.swift

@@ -0,0 +1,114 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+import Foundation
+import UIKit
+
+enum AudioRecorderRecordingState {
+    case recording
+    case stopped
+    case error
+}
+
+@MainActor
+class AudioRecorderState: ObservableObject, Identifiable {
+    let id = UUID()
+    
+    @Published private(set) var recordingState: AudioRecorderRecordingState = .stopped
+    @Published private(set) var duration = 0.0
+    @Published private(set) var waveform = Waveform(data: Array(repeating: 0, count: 100))
+    
+    private weak var audioRecorder: AudioRecorderProtocol?
+    private var cancellables: Set<AnyCancellable> = []
+    private var displayLink: CADisplayLink?
+    
+    func attachAudioRecorder(_ audioRecorder: AudioRecorderProtocol) {
+        if self.audioRecorder != nil {
+            detachAudioRecorder()
+        }
+        recordingState = .stopped
+        self.audioRecorder = audioRecorder
+        subscribeToAudioRecorder(audioRecorder)
+    }
+    
+    func detachAudioRecorder() {
+        guard audioRecorder != nil else { return }
+        audioRecorder?.stopRecording()
+        stopPublishUpdates()
+        cancellables = []
+        audioRecorder = nil
+        recordingState = .stopped
+    }
+    
+    func reportError(_ error: Error) {
+        recordingState = .error
+    }
+    
+    // MARK: - Private
+    
+    private func subscribeToAudioRecorder(_ audioRecorder: AudioRecorderProtocol) {
+        audioRecorder.actions
+            .receive(on: DispatchQueue.main)
+            .sink { [weak self] action in
+                guard let self else {
+                    return
+                }
+                self.handleAudioRecorderAction(action)
+            }
+            .store(in: &cancellables)
+    }
+    
+    private func handleAudioRecorderAction(_ action: AudioRecorderAction) {
+        switch action {
+        case .didStartRecording:
+            startPublishUpdates()
+            recordingState = .recording
+        case .didStopRecording:
+            stopPublishUpdates()
+            recordingState = .stopped
+        case .didFailWithError:
+            stopPublishUpdates()
+            recordingState = .stopped
+        }
+    }
+    
+    private func startPublishUpdates() {
+        if displayLink != nil {
+            stopPublishUpdates()
+        }
+        displayLink = CADisplayLink(target: self, selector: #selector(publishUpdate))
+        displayLink?.preferredFrameRateRange = .init(minimum: 10, maximum: 20)
+        displayLink?.add(to: .current, forMode: .common)
+    }
+    
+    @objc private func publishUpdate(displayLink: CADisplayLink) {
+        if let currentTime = audioRecorder?.currentTime {
+            duration = currentTime
+        }
+    }
+    
+    private func stopPublishUpdates() {
+        displayLink?.invalidate()
+        displayLink = nil
+    }
+}
+
+extension AudioRecorderState: Equatable {
+    nonisolated static func == (lhs: AudioRecorderState, rhs: AudioRecorderState) -> Bool {
+        lhs.id == rhs.id
+    }
+}

ElementX/Sources/Services/Room/RoomProxyProtocol.swift

@@ -115,7 +115,7 @@ protocol RoomProxyProtocol {
                    audioInfo: AudioInfo,
                    progressSubject: CurrentValueSubject<Double, Never>?,
                    requestHandle: @MainActor (SendAttachmentJoinHandleProtocol) -> Void) async -> Result<Void, RoomProxyError>
-    
+
     func sendFile(url: URL,
                   fileInfo: FileInfo,
                   progressSubject: CurrentValueSubject<Double, Never>?,

ElementX/Sources/Services/Timeline/TimelineController/MockRoomTimelineController.swift

@@ -67,7 +67,7 @@ class MockRoomTimelineController: RoomTimelineControllerProtocol {
     func processItemTap(_ itemID: TimelineItemIdentifier) async -> RoomTimelineControllerAction { .none }
     
     func sendMessage(_ message: String, html: String?, inReplyTo itemID: TimelineItemIdentifier?) async { }
-    
+        
     func toggleReaction(_ reaction: String, to itemID: TimelineItemIdentifier) async { }
 
     func editMessage(_ newMessage: String, html: String?, original itemID: TimelineItemIdentifier) async { }
@@ -90,6 +90,8 @@ class MockRoomTimelineController: RoomTimelineControllerProtocol {
     
     func playPauseAudio(for itemID: TimelineItemIdentifier) async { }
     
+    func pauseAudio() { }
+    
     func seekAudio(for itemID: TimelineItemIdentifier, progress: Double) async { }
     
     // MARK: - UI Test signalling

ElementX/Sources/Services/Timeline/TimelineController/RoomTimelineController.swift

@@ -240,6 +240,7 @@ class RoomTimelineController: RoomTimelineControllerProtocol {
         if let playerState = timelineAudioPlayerStates[itemID] {
             return playerState
         }
+        
         let playerState = AudioPlayerState(duration: voiceMessageRoomTimelineItem.content.duration,
                                            waveform: voiceMessageRoomTimelineItem.content.waveform)
         timelineAudioPlayerStates[itemID] = playerState
@@ -278,6 +279,7 @@ class RoomTimelineController: RoomTimelineControllerProtocol {
             // Load content
             do {
                 let url = try await voiceMessageMediaManager.loadVoiceMessageFromSource(source, body: nil)
+
                 // Make sure that the player is still attached, as it may have been detached while waiting for the voice message to be loaded.
                 if playerState.isAttached {
                     player.load(mediaSource: source, using: url)
@@ -297,10 +299,16 @@ class RoomTimelineController: RoomTimelineControllerProtocol {
         }
     }
     
+    func pauseAudio() {
+        timelineAudioPlayerStates.forEach { _, playerState in
+            playerState.detachAudioPlayer()
+        }
+    }
+    
     func seekAudio(for itemID: TimelineItemIdentifier, progress: Double) async {
         await timelineAudioPlayerStates[itemID]?.updateState(progress: progress)
     }
-
+    
     // MARK: - Private
     
     @objc private func contentSizeCategoryDidChange() {

ElementX/Sources/Services/Timeline/TimelineController/RoomTimelineControllerProtocol.swift

@@ -52,7 +52,7 @@ protocol RoomTimelineControllerProtocol {
     func sendReadReceipt(for itemID: TimelineItemIdentifier) async -> Result<Void, RoomTimelineControllerError>
     
     func sendMessage(_ message: String, html: String?, inReplyTo itemID: TimelineItemIdentifier?) async
-
+    
     func editMessage(_ newMessage: String, html: String?, original itemID: TimelineItemIdentifier) async
     
     func toggleReaction(_ reaction: String, to itemID: TimelineItemIdentifier) async
@@ -69,6 +69,8 @@ protocol RoomTimelineControllerProtocol {
     
     func playPauseAudio(for itemID: TimelineItemIdentifier) async
     
+    func pauseAudio()
+    
     func seekAudio(for itemID: TimelineItemIdentifier, progress: Double) async
 }
 

ElementX/Sources/Services/Timeline/TimelineItems/Items/Messages/VoiceMessages/VoiceMessageRoomPlaybackView.swift

@@ -18,16 +18,16 @@ import SwiftUI
 
 struct VoiceMessageRoomPlaybackView: View {
     @ObservedObject var playerState: AudioPlayerState
+    @ScaledMetric private var waveformLineWidth = 2.0
+    @ScaledMetric private var waveformLinePadding = 2.0
+
     let onPlayPause: () -> Void
     let onSeek: (Double) -> Void
     let onScrubbing: (Bool) -> Void
 
     private let feedbackGenerator = UIImpactFeedbackGenerator(style: .heavy)
     @State private var sendFeedback = false
-    
-    @ScaledMetric private var waveformLineWidth = 2.0
-    @ScaledMetric private var waveformLinePadding = 2.0
-    private let waveformMaxWidth: CGFloat = 150
+        
     @ScaledMetric private var playPauseButtonSize = 32
     @ScaledMetric private var playPauseImagePadding = 8
     
@@ -67,6 +67,7 @@ struct VoiceMessageRoomPlaybackView: View {
             HStack {
                 playPauseButton
                 Text(timeLabelContent)
+                    .lineLimit(1)
                     .font(.compound.bodySMSemibold)
                     .foregroundColor(.compound.textSecondary)
                     .monospacedDigit()
@@ -99,7 +100,6 @@ struct VoiceMessageRoomPlaybackView: View {
                             }
                         })
             }
-            .frame(maxWidth: waveformMaxWidth)
         }
         .onChange(of: dragState) { newDragState in
             switch newDragState {
@@ -139,6 +139,7 @@ struct VoiceMessageRoomPlaybackView: View {
                         .offset(x: playerState.playbackState == .playing ? 0 : 2)
                         .aspectRatio(contentMode: .fit)
                         .foregroundColor(.compound.iconSecondary)
+                        .accessibilityLabel(playerState.playbackState == .playing ? L10n.a11yPause : L10n.a11yPlay)
                 }
             }
         }

ElementX/Sources/Services/VoiceMessage/VoiceMessageMediaManager.swift

@@ -63,7 +63,7 @@ class VoiceMessageMediaManager: VoiceMessageMediaManagerProtocol {
         if let fileURL = voiceMessageCache.fileURL(for: source) {
             return fileURL
         }
-        
+                
         // Otherwise, load the file from source
         guard case .success(let fileHandle) = await mediaProvider.loadFileFromSource(source, body: body) else {
             throw MediaProviderError.failedRetrievingFile