Add voice message rendering to the timeline (#1776)

This commit is contained in:
Nicolas Mauri
2023-09-21 16:36:21 +02:00
committed by GitHub
parent eba3e25373
commit b8e5611c00
31 changed files with 660 additions and 20 deletions
@@ -82,6 +82,16 @@ class MockRoomTimelineController: RoomTimelineControllerProtocol {
func retryDecryption(for sessionID: String) async { }
func playbackViewState(for itemID: TimelineItemIdentifier) -> VoiceRoomPlaybackViewState? {
VoiceRoomPlaybackViewState(duration: 10.0,
waveform: nil,
progress: 0.0)
}
func playPauseAudio(for itemID: TimelineItemIdentifier) async { }
func seekAudio(for itemID: TimelineItemIdentifier, progress: Double) async { }
// MARK: - UI Test signalling
/// The cancellable used for UI Tests signalling.
@@ -36,7 +36,8 @@ class RoomTimelineController: RoomTimelineControllerProtocol {
let callbacks = PassthroughSubject<RoomTimelineControllerCallback, Never>()
private(set) var timelineItems = [RoomTimelineItemProtocol]()
private var timelineAudioPlaybackViewStates = [TimelineItemIdentifier: VoiceRoomPlaybackViewState]()
var roomID: String {
roomProxy.id
}
@@ -220,6 +221,34 @@ class RoomTimelineController: RoomTimelineControllerProtocol {
func retryDecryption(for sessionID: String) async {
await roomProxy.retryDecryption(for: sessionID)
}
func playbackViewState(for itemID: TimelineItemIdentifier) -> VoiceRoomPlaybackViewState? {
guard let timelineItem = timelineItems.firstUsingStableID(itemID) else {
MXLog.error("timelineItem not found")
return .none
}
switch timelineItem {
case let item as VoiceRoomTimelineItem:
if let playbackViewState = timelineAudioPlaybackViewStates[itemID] {
return playbackViewState
}
let playbackViewState = VoiceRoomPlaybackViewState(duration: item.content.duration,
waveform: item.content.waveform)
timelineAudioPlaybackViewStates[itemID] = playbackViewState
return playbackViewState
default:
return .none
}
}
func playPauseAudio(for itemID: TimelineItemIdentifier) async { }
func seekAudio(for itemID: TimelineItemIdentifier, progress: Double) async {
Task {
timelineAudioPlaybackViewStates[itemID]?.updateState(progress: progress)
}
}
// MARK: - Private
@@ -64,6 +64,12 @@ protocol RoomTimelineControllerProtocol {
func debugInfo(for itemID: TimelineItemIdentifier) -> TimelineItemDebugInfo
func retryDecryption(for sessionID: String) async
func playbackViewState(for itemID: TimelineItemIdentifier) -> VoiceRoomPlaybackViewState?
func playPauseAudio(for itemID: TimelineItemIdentifier) async
func seekAudio(for itemID: TimelineItemIdentifier, progress: Double) async
}
extension RoomTimelineControllerProtocol {
@@ -25,6 +25,7 @@ enum EventBasedMessageTimelineItemContentType: Hashable {
case text(TextRoomTimelineItemContent)
case video(VideoRoomTimelineItemContent)
case location(LocationRoomTimelineItemContent)
case voice(AudioRoomTimelineItemContent)
}
protocol EventBasedMessageTimelineItemProtocol: EventBasedTimelineItemProtocol {
@@ -20,6 +20,7 @@ import UniformTypeIdentifiers
struct AudioRoomTimelineItemContent: Hashable {
let body: String
let duration: TimeInterval
let waveform: Waveform?
let source: MediaSourceProxy?
let contentType: UTType?
}
@@ -0,0 +1,167 @@
//
// Copyright 2023 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import SwiftUI
struct VoiceRoomPlaybackView: View {
@ObservedObject var playbackViewState: VoiceRoomPlaybackViewState
private let feedbackGenerator = UIImpactFeedbackGenerator(style: .heavy)
@State private var sendFeedback = false
private let waveformMaxWidth: CGFloat = 150
private let playPauseButtonSize = CGSize(width: 32, height: 32)
private static let elapsedTimeFormatter: DateFormatter = {
let dateFormatter = DateFormatter()
dateFormatter.dateFormat = "m:ss"
return dateFormatter
}()
var onPlayPause: () -> Void = { }
var onSeek: (Double) -> Void = { _ in }
var onWaveformDragStateChanged: (Bool) -> Void = { _ in }
private enum DragState: Equatable {
case inactive
case pressing
case dragging(progress: Double)
var progress: Double {
switch self {
case .inactive, .pressing:
return .zero
case .dragging(let progress):
return progress
}
}
var isActive: Bool {
switch self {
case .inactive:
return false
case .pressing, .dragging:
return true
}
}
var isDragging: Bool {
switch self {
case .inactive, .pressing:
return false
case .dragging:
return true
}
}
}
@GestureState private var dragState = DragState.inactive
var timeLabelContent: String {
// Display the duration if progress is 0.0
let percent = playbackViewState.progress > 0.0 ? playbackViewState.progress : 1.0
return Self.elapsedTimeFormatter.string(from: Date(timeIntervalSinceReferenceDate: playbackViewState.duration * percent))
}
var body: some View {
HStack {
HStack {
playPauseButton
Text(timeLabelContent)
.font(.compound.bodySMSemibold)
.foregroundColor(.compound.textSecondary)
.monospacedDigit()
}
.padding(.vertical, 6)
GeometryReader { geometry in
WaveformView(waveform: playbackViewState.waveform, progress: playbackViewState.progress)
// Add a gesture to drag the waveform
.gesture(LongPressGesture()
.sequenced(before: DragGesture(coordinateSpace: .local))
.updating($dragState) { value, state, _ in
switch value {
// Long press begins.
case .first(true):
state = .pressing
// Long press confirmed, dragging may begin.
case .second(true, let drag):
let progress: Double = (drag?.location.x ?? .zero) / geometry.size.width
state = .dragging(progress: progress)
// Dragging ended or the long press cancelled.
default:
state = .inactive
}
})
}
.frame(maxWidth: waveformMaxWidth)
}
.onChange(of: dragState) { newDragState in
switch newDragState {
case .inactive:
onWaveformDragStateChanged(false)
case .pressing:
onWaveformDragStateChanged(true)
feedbackGenerator.prepare()
sendFeedback = true
case .dragging(let progress):
if sendFeedback {
feedbackGenerator.impactOccurred()
sendFeedback = false
}
if abs(progress - playbackViewState.progress) > 0.01 {
onSeek(max(0, min(progress, 1.0)))
}
}
}
.padding(.vertical, 2)
.padding(.horizontal, 8)
}
@ViewBuilder
var playPauseButton: some View {
Button {
onPlayPause()
} label: {
Image(systemName: playbackViewState.playing ? "pause.fill" : "play.fill")
.foregroundColor(.compound.iconSecondary)
.background(
Circle()
.frame(width: playPauseButtonSize.width,
height: playPauseButtonSize.height)
.foregroundColor(.compound.bgCanvasDefault)
)
.padding(.trailing, 7)
}
}
}
struct VoiceRoomPlaybackView_Previews: PreviewProvider, TestablePreview {
static let waveform = Waveform(data: [3, 127, 400, 266, 126, 122, 373, 251, 45, 112,
334, 205, 99, 138, 397, 354, 125, 361, 199, 51,
294, 131, 19, 2, 3, 3, 1, 2, 0, 0,
0, 0, 0, 0, 0, 3])
static let playbackViewState = VoiceRoomPlaybackViewState(duration: 10.0,
waveform: waveform,
progress: 0.3)
static var previews: some View {
VoiceRoomPlaybackView(playbackViewState: playbackViewState,
onPlayPause: { playbackViewState.updateState(playing: !playbackViewState.playing) },
onSeek: { playbackViewState.updateState(progress: $0) })
.fixedSize(horizontal: false, vertical: true)
}
}
@@ -0,0 +1,43 @@
//
// Copyright 2023 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import Combine
import Foundation
@MainActor
class VoiceRoomPlaybackViewState: ObservableObject {
let duration: Double
let waveform: Waveform
@Published private(set) var loading: Bool
@Published private(set) var playing: Bool
@Published private(set) var progress: Double
init(duration: Double = 0.0, waveform: Waveform? = nil, progress: Double = 0.0) {
self.duration = duration
self.waveform = waveform ?? Waveform(data: [])
self.progress = progress
loading = false
playing = false
}
func updateState(progress: Double) {
self.progress = max(0.0, min(progress, 1.0))
}
func updateState(playing: Bool) {
self.playing = playing
}
}
@@ -0,0 +1,92 @@
//
// Copyright 2023 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import Foundation
import SwiftUI
struct VoiceRoomTimelineView: View {
@EnvironmentObject private var context: RoomScreenViewModel.Context
let timelineItem: VoiceRoomTimelineItem
let playbackViewState: VoiceRoomPlaybackViewState
init(timelineItem: VoiceRoomTimelineItem, playbackViewState: VoiceRoomPlaybackViewState?) {
self.timelineItem = timelineItem
if playbackViewState == nil {
MXLog.error("[VoiceRoomTimelineView] Voice audio playback state is missing")
}
self.playbackViewState = playbackViewState ?? VoiceRoomPlaybackViewState()
}
var body: some View {
TimelineStyler(timelineItem: timelineItem) {
VoiceRoomPlaybackView(playbackViewState: playbackViewState,
onPlayPause: onPlaybackPlayPause,
onSeek: onPlaybackSeek(_:),
onWaveformDragStateChanged: onPlaybackDragStateChanged(_:))
.fixedSize(horizontal: false, vertical: true)
}
}
private func onPlaybackPlayPause() {
context.send(viewAction: .playPauseAudio(itemID: timelineItem.id))
}
private func onPlaybackSeek(_ progress: Double) {
context.send(viewAction: .seekAudio(itemID: timelineItem.id, progress: progress))
}
private func onPlaybackDragStateChanged(_ dragging: Bool) {
if dragging {
context.send(viewAction: .disableLongPress(itemID: timelineItem.id))
} else {
context.send(viewAction: .enableLongPress(itemID: timelineItem.id))
}
}
}
struct VoiceRoomTimelineView_Previews: PreviewProvider, TestablePreview {
static let viewModel = RoomScreenViewModel.mock
static let voiceRoomTimelineItem = VoiceRoomTimelineItem(id: .random,
timestamp: "Now",
isOutgoing: false,
isEditable: false,
isThreaded: false,
sender: .init(id: "Bob"),
content: .init(body: "audio.ogg",
duration: 300,
waveform: Waveform.mockWaveform,
source: nil,
contentType: nil))
static let playbackViewState = VoiceRoomPlaybackViewState(duration: 10.0,
waveform: Waveform.mockWaveform,
progress: 0.4)
static var previews: some View {
body.environmentObject(viewModel.context)
.previewDisplayName("Bubble")
body
.environment(\.timelineStyle, .plain)
.environmentObject(viewModel.context)
.previewDisplayName("Plain")
}
static var body: some View {
VoiceRoomTimelineView(timelineItem: voiceRoomTimelineItem, playbackViewState: playbackViewState)
.fixedSize(horizontal: false, vertical: true)
}
}
@@ -0,0 +1,88 @@
//
// Copyright 2023 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import SwiftUI
struct Waveform: Equatable, Hashable {
let data: [UInt16]
}
extension Waveform {
func normalisedData(count: Int) -> [Float] {
guard count > 0 else {
return []
}
let stride = max(1, Int(data.count / count))
let data = data.striding(by: stride)
let max = data.max().flatMap { Float($0) } ?? 0
return data.map { Float($0) / max }
}
}
extension Waveform {
static let mockWaveform = Waveform(data: [0, 0, 0, 3, 3, 127, 400, 266, 126, 122, 373, 251, 45, 112,
334, 205, 99, 138, 397, 354, 125, 361, 199, 51,
294, 131, 19, 2, 3, 3, 1, 2, 0, 0,
0, 0])
}
struct WaveformView: View {
let lineWidth: CGFloat = 2
let linePadding: CGFloat = 2
var waveform: Waveform
let minimumGraphAmplitude: CGFloat = 1
var progress: CGFloat = 0.0
var body: some View {
GeometryReader { geometry in
ZStack(alignment: .leading) {
Rectangle().fill(Color.compound.iconQuaternary)
.frame(width: geometry.size.width, height: geometry.size.height)
Rectangle().fill(Color.compound.iconSecondary)
.frame(width: max(0.0, geometry.size.width * progress), height: geometry.size.height)
}
.mask(alignment: .leading) {
Path { path in
let width = geometry.size.width
let height = geometry.size.height
let centerY = geometry.size.height / 2
let visibleSamplesCount = Int(width / (lineWidth + linePadding))
let normalisedData = waveform.normalisedData(count: visibleSamplesCount)
var xOffset: CGFloat = lineWidth / 2
var index = 0
while xOffset < width - lineWidth {
let sample = CGFloat(index >= normalisedData.count ? 0 : normalisedData[index])
let drawingAmplitude = max(minimumGraphAmplitude, sample * height)
path.move(to: CGPoint(x: xOffset, y: centerY - drawingAmplitude / 2))
path.addLine(to: CGPoint(x: xOffset, y: centerY + drawingAmplitude / 2))
xOffset += lineWidth + linePadding
index += 1
}
}
.stroke(Color.compound.iconSecondary, style: StrokeStyle(lineWidth: lineWidth, lineCap: .round))
}
}
}
}
struct WaveformView_Previews: PreviewProvider, TestablePreview {
static var previews: some View {
WaveformView(waveform: Waveform.mockWaveform)
.frame(width: 140, height: 50)
}
}
@@ -0,0 +1,40 @@
//
// Copyright 2023 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import Foundation
struct VoiceRoomTimelineItem: EventBasedMessageTimelineItemProtocol, Equatable {
let id: TimelineItemIdentifier
let timestamp: String
let isOutgoing: Bool
let isEditable: Bool
let isThreaded: Bool
let sender: TimelineItemSender
let content: AudioRoomTimelineItemContent
var replyDetails: TimelineItemReplyDetails?
var properties = RoomTimelineItemProperties()
var body: String {
content.body
}
var contentType: EventBasedMessageTimelineItemContentType {
.voice(content)
}
}
@@ -22,6 +22,7 @@ struct RoomTimelineItemFactory: RoomTimelineItemFactoryProtocol {
private let mediaProvider: MediaProviderProtocol
private let attributedStringBuilder: AttributedStringBuilderProtocol
private let stateEventStringBuilder: RoomStateEventStringBuilder
private let appSettings: AppSettings
/// The Matrix ID of the current user.
private let userID: String
@@ -29,11 +30,13 @@ struct RoomTimelineItemFactory: RoomTimelineItemFactoryProtocol {
init(userID: String,
mediaProvider: MediaProviderProtocol,
attributedStringBuilder: AttributedStringBuilderProtocol,
stateEventStringBuilder: RoomStateEventStringBuilder) {
stateEventStringBuilder: RoomStateEventStringBuilder,
appSettings: AppSettings) {
self.userID = userID
self.mediaProvider = mediaProvider
self.attributedStringBuilder = attributedStringBuilder
self.stateEventStringBuilder = stateEventStringBuilder
self.appSettings = appSettings
}
func buildTimelineItem(for eventItemProxy: EventTimelineItemProxy) -> RoomTimelineItemProtocol? {
@@ -95,7 +98,11 @@ struct RoomTimelineItemFactory: RoomTimelineItemFactoryProtocol {
case .emote(content: let content):
return buildEmoteTimelineItem(for: eventItemProxy, messageTimelineItem, content, isOutgoing, isThreaded)
case .audio(let content):
return buildAudioTimelineItem(for: eventItemProxy, messageTimelineItem, content, isOutgoing, isThreaded)
if appSettings.voiceMessageEnabled, content.voice != nil {
return buildVoiceTimelineItem(for: eventItemProxy, messageTimelineItem, content, isOutgoing, isThreaded)
} else {
return buildAudioTimelineItem(for: eventItemProxy, messageTimelineItem, content, isOutgoing, isThreaded)
}
case .location(let content):
return buildLocationTimelineItem(for: eventItemProxy, messageTimelineItem, content, isOutgoing, isThreaded)
case .none:
@@ -256,6 +263,25 @@ struct RoomTimelineItemFactory: RoomTimelineItemFactoryProtocol {
orderedReadReceipts: orderReadReceipts(eventItemProxy.readReceipts)))
}
private func buildVoiceTimelineItem(for eventItemProxy: EventTimelineItemProxy,
_ messageTimelineItem: Message,
_ messageContent: AudioMessageContent,
_ isOutgoing: Bool,
_ isThreaded: Bool) -> RoomTimelineItemProtocol {
VoiceRoomTimelineItem(id: eventItemProxy.id,
timestamp: eventItemProxy.timestamp.formatted(date: .omitted, time: .shortened),
isOutgoing: isOutgoing,
isEditable: eventItemProxy.isEditable,
isThreaded: isThreaded,
sender: eventItemProxy.sender,
content: buildAudioTimelineItemContent(messageContent),
replyDetails: buildReplyToDetailsFrom(details: messageTimelineItem.inReplyTo()),
properties: RoomTimelineItemProperties(isEdited: messageTimelineItem.isEdited(),
reactions: aggregateReactions(eventItemProxy.reactions),
deliveryStatus: eventItemProxy.deliveryStatus,
orderedReadReceipts: orderReadReceipts(eventItemProxy.readReceipts)))
}
private func buildFileTimelineItem(for eventItemProxy: EventTimelineItemProxy,
_ messageTimelineItem: Message,
_ messageContent: FileMessageContent,
@@ -428,10 +454,16 @@ struct RoomTimelineItemFactory: RoomTimelineItemFactoryProtocol {
}
private func buildAudioTimelineItemContent(_ messageContent: AudioMessageContent) -> AudioRoomTimelineItemContent {
AudioRoomTimelineItemContent(body: messageContent.body,
duration: messageContent.info?.duration ?? 0,
source: MediaSourceProxy(source: messageContent.source, mimeType: messageContent.info?.mimetype),
contentType: UTType(mimeType: messageContent.info?.mimetype, fallbackFilename: messageContent.body))
var waveform: Waveform?
if let audioWaveform = messageContent.audio?.waveform {
waveform = Waveform(data: audioWaveform)
}
return AudioRoomTimelineItemContent(body: messageContent.body,
duration: messageContent.info?.duration ?? 0,
waveform: waveform,
source: MediaSourceProxy(source: messageContent.source, mimeType: messageContent.info?.mimetype),
contentType: UTType(mimeType: messageContent.info?.mimetype, fallbackFilename: messageContent.body))
}
private func buildImageTimelineItemContent(_ messageContent: ImageMessageContent) -> ImageRoomTimelineItemContent {
@@ -583,7 +615,11 @@ struct RoomTimelineItemFactory: RoomTimelineItemFactoryProtocol {
let replyContent: EventBasedMessageTimelineItemContentType
switch timelineItem.asMessage()?.msgtype() {
case .audio(let content):
replyContent = .audio(buildAudioTimelineItemContent(content))
if appSettings.voiceMessageEnabled, content.voice != nil {
replyContent = .voice(buildAudioTimelineItemContent(content))
} else {
replyContent = .audio(buildAudioTimelineItemContent(content))
}
case .emote(let content):
replyContent = .emote(buildEmoteTimelineItemContent(senderDisplayName: sender.displayName, senderID: sender.id, messageContent: content))
case .file(let content):
@@ -74,6 +74,8 @@ struct RoomTimelineItemView: View {
LocationRoomTimelineView(timelineItem: item)
case .poll(let item):
PollRoomTimelineView(timelineItem: item)
case .voice(let item):
VoiceRoomTimelineView(timelineItem: item, playbackViewState: context.viewState.audioPlaybackViewStateProvider?(item.id))
}
}
@@ -69,6 +69,7 @@ enum RoomTimelineItemType: Equatable {
case group(CollapsibleTimelineItem)
case location(LocationRoomTimelineItem)
case poll(PollRoomTimelineItem)
case voice(VoiceRoomTimelineItem)
init(item: RoomTimelineItemProtocol) {
switch item {
@@ -112,6 +113,8 @@ enum RoomTimelineItemType: Equatable {
self = .location(item)
case let item as PollRoomTimelineItem:
self = .poll(item)
case let item as VoiceRoomTimelineItem:
self = .voice(item)
default:
fatalError("Unknown timeline item")
}
@@ -138,7 +141,8 @@ enum RoomTimelineItemType: Equatable {
.state(let item as RoomTimelineItemProtocol),
.group(let item as RoomTimelineItemProtocol),
.location(let item as RoomTimelineItemProtocol),
.poll(let item as RoomTimelineItemProtocol):
.poll(let item as RoomTimelineItemProtocol),
.voice(let item as RoomTimelineItemProtocol):
return item.id
}
}
@@ -146,7 +150,7 @@ enum RoomTimelineItemType: Equatable {
/// Whether or not it is possible to send a reaction to this timeline item.
var isReactable: Bool {
switch self {
case .text, .image, .video, .audio, .file, .emote, .notice, .sticker, .location, .poll:
case .text, .image, .video, .audio, .file, .emote, .notice, .sticker, .location, .poll, .voice:
return true
case .redacted, .encrypted, .unsupported, .state: // Event based items that aren't reactable
return false