Add UI to record a voice message from the composer toolbar (#1892)

2026-11-03 16:10:20 +00:00 · 2023-10-16 17:18:51 +02:00
parent 0d72554e6f
commit 9ab3b20a49
43 changed files with 1241 additions and 87 deletions
@@ -0,0 +1,49 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import AVFoundation
+import Foundation
+import SwiftOGG
+
+enum AudioConverterError: Error {
+    case conversionFailed(Error?)
+}
+
+enum AudioConverterPreferredFileExtension: String {
+    case mpeg4aac = "m4a"
+    case ogg
+}
+
+struct AudioConverter: AudioConverterProtocol {
+    func convertToOpusOgg(sourceURL: URL, destinationURL: URL) throws {
+        do {
+            MXLog.debug("converting \(sourceURL) to \(destinationURL)")
+            try OGGConverter.convertM4aFileToOpusOGG(src: sourceURL, dest: destinationURL)
+        } catch {
+            MXLog.error("failed to convert to OpusOgg: \(error)")
+            throw AudioConverterError.conversionFailed(error)
+        }
+    }
+    
+    func convertToMPEG4AAC(sourceURL: URL, destinationURL: URL) throws {
+        do {
+            try OGGConverter.convertOpusOGGToM4aFile(src: sourceURL, dest: destinationURL)
+        } catch {
+            MXLog.error("failed to convert to MPEG4AAC: \(error)")
+            throw AudioConverterError.conversionFailed(error)
+        }
+    }
+}
@@ -0,0 +1,25 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+
+protocol AudioConverterProtocol {
+    func convertToOpusOgg(sourceURL: URL, destinationURL: URL) throws
+    func convertToMPEG4AAC(sourceURL: URL, destinationURL: URL) throws
+}
+
+// sourcery: AutoMockable
+extension AudioConverterProtocol { }
@@ -0,0 +1,223 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import AVFoundation
+import Combine
+import Foundation
+import UIKit
+
+private enum InternalAudioPlayerState {
+    case none
+    case loading
+    case readyToPlay
+    case playing
+    case paused
+    case stopped
+    case finishedPlaying
+    case error(Error)
+}
+
+enum AudioPlayerError: Error {
+    case genericError
+    case loadFileError
+}
+
+class AudioPlayer: NSObject, AudioPlayerProtocol {
+    var mediaSource: MediaSourceProxy?
+    
+    private var playerItem: AVPlayerItem?
+    private var internalAudioPlayer: AVQueuePlayer?
+    
+    private var cancellables = Set<AnyCancellable>()
+    private let actionsSubject: PassthroughSubject<AudioPlayerAction, Never> = .init()
+    var actions: AnyPublisher<AudioPlayerAction, Never> {
+        actionsSubject.eraseToAnyPublisher()
+    }
+    
+    private var internalState = InternalAudioPlayerState.none
+    
+    private var statusObserver: NSKeyValueObservation?
+    private var rateObserver: NSKeyValueObservation?
+    private var playToEndObserver: NSObjectProtocol?
+    private var appBackgroundObserver: NSObjectProtocol?
+    
+    private(set) var url: URL?
+    
+    var duration: TimeInterval {
+        abs(CMTimeGetSeconds(internalAudioPlayer?.currentItem?.duration ?? .zero))
+    }
+    
+    var currentTime: TimeInterval {
+        let currentTime = abs(CMTimeGetSeconds(internalAudioPlayer?.currentTime() ?? .zero))
+        return currentTime.isFinite ? currentTime : .zero
+    }
+    
+    var state: MediaPlayerState {
+        if case .loading = internalState {
+            return .loading
+        }
+        if case .stopped = internalState {
+            return .stopped
+        }
+        if case .playing = internalState {
+            return .playing
+        }
+        if case .paused = internalState {
+            return .paused
+        }
+        if case .error = internalState {
+            return .error
+        }
+        return .stopped
+    }
+    
+    private var isStopped = true
+    
+    deinit {
+        stop()
+        unloadContent()
+    }
+    
+    func load(mediaSource: MediaSourceProxy, using url: URL) {
+        unloadContent()
+        setInternalState(.loading)
+        self.mediaSource = mediaSource
+        self.url = url
+        playerItem = AVPlayerItem(url: url)
+        internalAudioPlayer = AVQueuePlayer(playerItem: playerItem)
+        addObservers()
+    }
+    
+    func play() {
+        isStopped = false
+        do {
+            try AVAudioSession.sharedInstance().setCategory(AVAudioSession.Category.playback)
+            try AVAudioSession.sharedInstance().setActive(true)
+        } catch {
+            MXLog.error("Could not redirect audio playback to speakers.")
+        }
+        internalAudioPlayer?.play()
+    }
+    
+    func pause() {
+        guard case .playing = internalState else { return }
+        internalAudioPlayer?.pause()
+    }
+    
+    func stop() {
+        guard !isStopped else { return }
+        isStopped = true
+        internalAudioPlayer?.pause()
+        internalAudioPlayer?.seek(to: .zero)
+    }
+    
+    func seek(to progress: Double) async {
+        guard let internalAudioPlayer else { return }
+        let time = progress * duration
+        await internalAudioPlayer.seek(to: CMTime(seconds: time, preferredTimescale: 60))
+    }
+    
+    // MARK: - Private
+    
+    private func unloadContent() {
+        mediaSource = nil
+        url = nil
+        internalAudioPlayer?.replaceCurrentItem(with: nil)
+        internalAudioPlayer = nil
+        playerItem = nil
+        removeObservers()
+    }
+
+    private func addObservers() {
+        guard let internalAudioPlayer, let playerItem else {
+            return
+        }
+        
+        statusObserver = playerItem.observe(\.status, options: [.old, .new]) { [weak self] _, _ in
+            guard let self else { return }
+            
+            switch playerItem.status {
+            case .failed:
+                self.setInternalState(.error(playerItem.error ?? AudioPlayerError.genericError))
+            case .readyToPlay:
+                guard state == .loading else { return }
+                self.setInternalState(.readyToPlay)
+            default:
+                break
+            }
+        }
+                
+        rateObserver = internalAudioPlayer.observe(\.rate, options: [.old, .new]) { [weak self] _, _ in
+            guard let self else { return }
+            
+            if internalAudioPlayer.rate == 0 {
+                if self.isStopped {
+                    self.setInternalState(.stopped)
+                } else {
+                    self.setInternalState(.paused)
+                }
+            } else {
+                self.setInternalState(.playing)
+            }
+        }
+                
+        NotificationCenter.default.publisher(for: Notification.Name.AVPlayerItemDidPlayToEndTime)
+            .sink { [weak self] _ in
+                guard let self else { return }
+                self.setInternalState(.finishedPlaying)
+            }
+            .store(in: &cancellables)
+        
+        // Pause playback uppon UIApplication.didBecomeActiveNotification notification
+        NotificationCenter.default.publisher(for: UIApplication.didEnterBackgroundNotification)
+            .sink { [weak self] _ in
+                guard let self else { return }
+                self.pause()
+            }
+            .store(in: &cancellables)
+    }
+    
+    private func removeObservers() {
+        statusObserver?.invalidate()
+        rateObserver?.invalidate()
+        cancellables.removeAll()
+    }
+    
+    private func setInternalState(_ state: InternalAudioPlayerState) {
+        internalState = state
+        switch state {
+        case .none:
+            break
+        case .loading:
+            actionsSubject.send(.didStartLoading)
+        case .readyToPlay:
+            actionsSubject.send(.didFinishLoading)
+            play()
+        case .playing:
+            actionsSubject.send(.didStartPlaying)
+        case .paused:
+            actionsSubject.send(.didPausePlaying)
+        case .stopped:
+            actionsSubject.send(.didStopPlaying)
+        case .finishedPlaying:
+            actionsSubject.send(.didFinishPlaying)
+            unloadContent()
+        case .error(let error):
+            MXLog.error("audio player did fail. \(error)")
+            actionsSubject.send(.didFailWithError(error: error))
+        }
+    }
+}
@@ -0,0 +1,35 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+import Foundation
+
+enum AudioPlayerAction {
+    case didStartLoading
+    case didFinishLoading
+    case didStartPlaying
+    case didPausePlaying
+    case didStopPlaying
+    case didFinishPlaying
+    case didFailWithError(error: Error)
+}
+
+protocol AudioPlayerProtocol: MediaPlayerProtocol {
+    var actions: AnyPublisher<AudioPlayerAction, Never> { get }
+}
+
+// sourcery: AutoMockable
+extension AudioPlayerProtocol { }
@@ -0,0 +1,159 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+import Foundation
+import UIKit
+
+enum AudioPlayerPlaybackState {
+    case loading
+    case readyToPlay
+    case playing
+    case stopped
+    case error
+}
+
+@MainActor
+class AudioPlayerState: ObservableObject, Identifiable {
+    let id = UUID()
+    let duration: Double
+    let waveform: Waveform
+    @Published private(set) var playbackState: AudioPlayerPlaybackState
+    @Published private(set) var progress: Double
+
+    private weak var audioPlayer: AudioPlayerProtocol?
+    private var cancellables: Set<AnyCancellable> = []
+    private var displayLink: CADisplayLink?
+
+    var isAttached: Bool {
+        audioPlayer != nil
+    }
+    
+    var isPublishingProgress: Bool {
+        displayLink != nil
+    }
+
+    init(duration: Double, waveform: Waveform? = nil, progress: Double = 0.0) {
+        self.duration = duration
+        self.waveform = waveform ?? Waveform(data: [])
+        self.progress = progress
+        playbackState = .stopped
+    }
+    
+    deinit {
+        displayLink?.invalidate()
+        displayLink = nil
+    }
+    
+    func updateState(progress: Double) async {
+        let progress = max(0.0, min(progress, 1.0))
+        self.progress = progress
+        if let audioPlayer {
+            await audioPlayer.seek(to: progress)
+        }
+    }
+    
+    func attachAudioPlayer(_ audioPlayer: AudioPlayerProtocol) {
+        if self.audioPlayer != nil {
+            detachAudioPlayer()
+        }
+        playbackState = .loading
+        self.audioPlayer = audioPlayer
+        subscribeToAudioPlayer(audioPlayer: audioPlayer)
+    }
+    
+    func detachAudioPlayer() {
+        guard audioPlayer != nil else { return }
+        audioPlayer?.stop()
+        stopPublishProgress()
+        cancellables = []
+        audioPlayer = nil
+        playbackState = .stopped
+    }
+    
+    func reportError(_ error: Error) {
+        playbackState = .error
+    }
+    
+    // MARK: - Private
+    
+    private func subscribeToAudioPlayer(audioPlayer: AudioPlayerProtocol) {
+        audioPlayer.actions
+            .receive(on: DispatchQueue.main)
+            .sink { [weak self] action in
+                guard let self else {
+                    return
+                }
+                Task {
+                    await self.handleAudioPlayerAction(action)
+                }
+            }
+            .store(in: &cancellables)
+    }
+    
+    private func handleAudioPlayerAction(_ action: AudioPlayerAction) async {
+        switch action {
+        case .didStartLoading:
+            playbackState = .loading
+        case .didFinishLoading:
+            playbackState = .readyToPlay
+        case .didStartPlaying:
+            if let audioPlayer {
+                await restoreAudioPlayerState(audioPlayer: audioPlayer)
+            }
+            startPublishProgress()
+            playbackState = .playing
+        case .didPausePlaying, .didStopPlaying, .didFinishPlaying:
+            stopPublishProgress()
+            playbackState = .stopped
+            if case .didFinishPlaying = action {
+                progress = 0.0
+            }
+        case .didFailWithError:
+            stopPublishProgress()
+        }
+    }
+    
+    private func startPublishProgress() {
+        if displayLink != nil {
+            stopPublishProgress()
+        }
+        displayLink = CADisplayLink(target: self, selector: #selector(updateProgress))
+        displayLink?.preferredFrameRateRange = .init(minimum: 10, maximum: 20)
+        displayLink?.add(to: .current, forMode: .common)
+    }
+    
+    @objc private func updateProgress(displayLink: CADisplayLink) {
+        if let currentTime = audioPlayer?.currentTime, duration > 0 {
+            progress = currentTime / duration
+        }
+    }
+    
+    private func stopPublishProgress() {
+        displayLink?.invalidate()
+        displayLink = nil
+    }
+    
+    private func restoreAudioPlayerState(audioPlayer: AudioPlayerProtocol) async {
+        await audioPlayer.seek(to: progress)
+    }
+}
+
+extension AudioPlayerState: Equatable {
+    nonisolated static func == (lhs: AudioPlayerState, rhs: AudioPlayerState) -> Bool {
+        lhs.id == rhs.id
+    }
+}
@@ -0,0 +1,115 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import AVFoundation
+import Combine
+import Foundation
+
+enum AudioRecorderError: Error {
+    case genericError
+}
+
+class AudioRecorder: NSObject, AudioRecorderProtocol, AVAudioRecorderDelegate {
+    private let silenceThreshold: Float = -50.0
+    
+    private var audioRecorder: AVAudioRecorder?
+    
+    private let actionsSubject: PassthroughSubject<AudioRecorderAction, Never> = .init()
+    var actions: AnyPublisher<AudioRecorderAction, Never> {
+        actionsSubject.eraseToAnyPublisher()
+    }
+    
+    var url: URL? {
+        audioRecorder?.url
+    }
+    
+    var currentTime: TimeInterval {
+        audioRecorder?.currentTime ?? 0
+    }
+    
+    var isRecording: Bool {
+        audioRecorder?.isRecording ?? false
+    }
+    
+    func recordWithOutputURL(_ url: URL) {
+        let settings = [AVFormatIDKey: Int(kAudioFormatMPEG4AAC),
+                        AVSampleRateKey: 48000,
+                        AVEncoderBitRateKey: 128_000,
+                        AVNumberOfChannelsKey: 1,
+                        AVEncoderAudioQualityKey: AVAudioQuality.high.rawValue]
+        
+        do {
+            try AVAudioSession.sharedInstance().setCategory(.playAndRecord, mode: .default)
+            try AVAudioSession.sharedInstance().setActive(true)
+            audioRecorder = try AVAudioRecorder(url: url, settings: settings)
+            audioRecorder?.delegate = self
+            audioRecorder?.isMeteringEnabled = true
+            audioRecorder?.record()
+            actionsSubject.send(.didStartRecording)
+        } catch {
+            MXLog.error("audio recording failed: \(error)")
+            actionsSubject.send(.didFailWithError(error: error))
+        }
+    }
+
+    func stopRecording() {
+        audioRecorder?.stop()
+
+        do {
+            try AVAudioSession.sharedInstance().setActive(false)
+        } catch {
+            actionsSubject.send(.didFailWithError(error: error))
+        }
+    }
+    
+    func peakPowerForChannelNumber(_ channelNumber: Int) -> Float {
+        guard isRecording, let audioRecorder else {
+            return 0.0
+        }
+        
+        audioRecorder.updateMeters()
+
+        return normalizedPowerLevelFromDecibels(audioRecorder.peakPower(forChannel: channelNumber))
+    }
+    
+    func averagePowerForChannelNumber(_ channelNumber: Int) -> Float {
+        guard isRecording, let audioRecorder else {
+            return 0.0
+        }
+        
+        audioRecorder.updateMeters()
+        
+        return normalizedPowerLevelFromDecibels(audioRecorder.averagePower(forChannel: channelNumber))
+    }
+        
+    // MARK: - AVAudioRecorderDelegate
+    
+    func audioRecorderDidFinishRecording(_ recorder: AVAudioRecorder, successfully success: Bool) {
+        if success {
+            actionsSubject.send(.didStopRecording)
+        } else {
+            actionsSubject.send(.didFailWithError(error: AudioRecorderError.genericError))
+        }
+    }
+    
+    func audioRecorderEncodeErrorDidOccur(_ recorder: AVAudioRecorder, error: Error?) {
+        actionsSubject.send(.didFailWithError(error: error ?? AudioRecorderError.genericError))
+    }
+    
+    private func normalizedPowerLevelFromDecibels(_ decibels: Float) -> Float {
+        decibels / silenceThreshold
+    }
+}
@@ -0,0 +1,38 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+import Foundation
+
+enum AudioRecorderAction {
+    case didStartRecording
+    case didStopRecording
+    case didFailWithError(error: Error)
+}
+
+protocol AudioRecorderProtocol: AnyObject {
+    var actions: AnyPublisher<AudioRecorderAction, Never> { get }
+    var currentTime: TimeInterval { get }
+    var isRecording: Bool { get }
+    var url: URL? { get }
+    
+    func recordWithOutputURL(_ url: URL)
+    func stopRecording()
+    func averagePowerForChannelNumber(_ channelNumber: Int) -> Float
+}
+
+// sourcery: AutoMockable
+extension AudioRecorderProtocol { }
@@ -0,0 +1,114 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+import Foundation
+import UIKit
+
+enum AudioRecorderRecordingState {
+    case recording
+    case stopped
+    case error
+}
+
+@MainActor
+class AudioRecorderState: ObservableObject, Identifiable {
+    let id = UUID()
+    
+    @Published private(set) var recordingState: AudioRecorderRecordingState = .stopped
+    @Published private(set) var duration = 0.0
+    @Published private(set) var waveform = Waveform(data: Array(repeating: 0, count: 100))
+    
+    private weak var audioRecorder: AudioRecorderProtocol?
+    private var cancellables: Set<AnyCancellable> = []
+    private var displayLink: CADisplayLink?
+    
+    func attachAudioRecorder(_ audioRecorder: AudioRecorderProtocol) {
+        if self.audioRecorder != nil {
+            detachAudioRecorder()
+        }
+        recordingState = .stopped
+        self.audioRecorder = audioRecorder
+        subscribeToAudioRecorder(audioRecorder)
+    }
+    
+    func detachAudioRecorder() {
+        guard audioRecorder != nil else { return }
+        audioRecorder?.stopRecording()
+        stopPublishUpdates()
+        cancellables = []
+        audioRecorder = nil
+        recordingState = .stopped
+    }
+    
+    func reportError(_ error: Error) {
+        recordingState = .error
+    }
+    
+    // MARK: - Private
+    
+    private func subscribeToAudioRecorder(_ audioRecorder: AudioRecorderProtocol) {
+        audioRecorder.actions
+            .receive(on: DispatchQueue.main)
+            .sink { [weak self] action in
+                guard let self else {
+                    return
+                }
+                self.handleAudioRecorderAction(action)
+            }
+            .store(in: &cancellables)
+    }
+    
+    private func handleAudioRecorderAction(_ action: AudioRecorderAction) {
+        switch action {
+        case .didStartRecording:
+            startPublishUpdates()
+            recordingState = .recording
+        case .didStopRecording:
+            stopPublishUpdates()
+            recordingState = .stopped
+        case .didFailWithError:
+            stopPublishUpdates()
+            recordingState = .stopped
+        }
+    }
+    
+    private func startPublishUpdates() {
+        if displayLink != nil {
+            stopPublishUpdates()
+        }
+        displayLink = CADisplayLink(target: self, selector: #selector(publishUpdate))
+        displayLink?.preferredFrameRateRange = .init(minimum: 10, maximum: 20)
+        displayLink?.add(to: .current, forMode: .common)
+    }
+    
+    @objc private func publishUpdate(displayLink: CADisplayLink) {
+        if let currentTime = audioRecorder?.currentTime {
+            duration = currentTime
+        }
+    }
+    
+    private func stopPublishUpdates() {
+        displayLink?.invalidate()
+        displayLink = nil
+    }
+}
+
+extension AudioRecorderState: Equatable {
+    nonisolated static func == (lhs: AudioRecorderState, rhs: AudioRecorderState) -> Bool {
+        lhs.id == rhs.id
+    }
+}