Implement support for ElementCall through Rust side widgets (#1906)

* Implement support for ElementCall through Rust side widgets

* Add Element Call feature flag

* Fix the unit tests

* Address PR comments
This commit is contained in:
Stefan Ceriu
2023-10-17 12:41:56 +03:00
committed by GitHub
parent 524e4b3f1e
commit 6e8dff2ef6
25 changed files with 1059 additions and 18 deletions
@@ -0,0 +1,65 @@
//
// Copyright 2022 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import Combine
import SwiftUI
struct CallScreenCoordinatorParameters {
let roomProxy: RoomProxyProtocol
/// Which Element Call instance should be used
let callBaseURL: URL
/// A way to identify the current client against Element Call
let clientID: String
}
enum CallScreenCoordinatorAction {
case dismiss
}
final class CallScreenCoordinator: CoordinatorProtocol {
private let parameters: CallScreenCoordinatorParameters
private var viewModel: CallScreenViewModelProtocol
private let actionsSubject: PassthroughSubject<CallScreenCoordinatorAction, Never> = .init()
private var cancellables: Set<AnyCancellable> = .init()
var actions: AnyPublisher<CallScreenCoordinatorAction, Never> {
actionsSubject.eraseToAnyPublisher()
}
init(parameters: CallScreenCoordinatorParameters) {
self.parameters = parameters
viewModel = CallScreenViewModel(roomProxy: parameters.roomProxy,
callBaseURL: parameters.callBaseURL,
clientID: parameters.clientID)
}
func start() {
viewModel.actions.sink { [weak self] action in
guard let self else { return }
switch action {
case .dismiss:
actionsSubject.send(.dismiss)
}
}
.store(in: &cancellables)
}
func toPresentable() -> AnyView {
AnyView(CallScreen(context: viewModel.context))
}
}
@@ -0,0 +1,39 @@
//
// Copyright 2022 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import Foundation
enum CallScreenViewModelAction {
case dismiss
}
struct CallScreenViewState: BindableState {
let messageHandler: String
let script: String?
var url: URL?
var bindings = Bindings()
}
struct Bindings {
var javaScriptMessageHandler: ((Any) -> Void)?
var javaScriptEvaluator: ((String) async throws -> Any)?
var alertInfo: AlertInfo<UUID>?
}
enum CallScreenViewAction {
case urlChanged(URL?)
}
@@ -0,0 +1,206 @@
//
// Copyright 2022 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import AVFoundation
import CallKit
import Combine
import SwiftUI
typealias CallScreenViewModelType = StateStoreViewModel<CallScreenViewState, CallScreenViewAction>
class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol {
private let roomProxy: RoomProxyProtocol
private let callBaseURL: URL
private let clientID: String
private let widgetDriver: ElementCallWidgetDriverProtocol
private let callController = CXCallController()
private let callProvider = CXProvider(configuration: .init())
private let callID = UUID()
private let actionsSubject: PassthroughSubject<CallScreenViewModelAction, Never> = .init()
var actions: AnyPublisher<CallScreenViewModelAction, Never> {
actionsSubject.eraseToAnyPublisher()
}
deinit {
tearDownVoIPSession(callID: callID)
}
/// Designated initialiser
/// - Parameters:
/// - roomProxy: The room in which the call should be created
/// - callBaseURL: Which Element Call instance should be used
/// - clientID: Something to identify the current client on the Element Call side
init(roomProxy: RoomProxyProtocol, callBaseURL: URL, clientID: String) {
self.roomProxy = roomProxy
self.callBaseURL = callBaseURL
self.clientID = clientID
widgetDriver = roomProxy.elementCallWidgetDriver()
super.init(initialViewState: CallScreenViewState(messageHandler: Self.eventHandlerName,
script: Self.eventHandlerInjectionScript))
state.bindings.javaScriptMessageHandler = { [weak self] message in
guard let self,
let message = message as? String else {
return
}
Task {
await self.widgetDriver.sendMessage(message)
}
}
widgetDriver.messagePublisher
.receive(on: DispatchQueue.main)
.sink { [weak self] receivedMessage in
guard let self else { return }
Task {
do {
let message = "postMessage(\(receivedMessage), '*')"
let result = try await self.state.bindings.javaScriptEvaluator?(message)
MXLog.debug("Evaluated javascript: \(message) with result: \(String(describing: result))")
} catch {
MXLog.error("Received javascript evaluation error: \(error)")
}
}
}
.store(in: &cancellables)
widgetDriver.actions
.receive(on: DispatchQueue.main)
.sink { [weak self] action in
guard let self else { return }
switch action {
case .callEnded:
actionsSubject.send(.dismiss)
default:
break
}
}
.store(in: &cancellables)
Task {
switch await widgetDriver.start(baseURL: callBaseURL, clientID: clientID) {
case .success(let url):
state.url = url
case .failure(let error):
MXLog.error("Failed starting ElementCall Widget Driver with error: \(error)")
state.bindings.alertInfo = .init(id: UUID(), title: L10n.errorUnknown, primaryButton: .init(title: L10n.actionOk, action: { [weak self] in
self?.actionsSubject.send(.dismiss)
}))
return
}
do {
try await setupVoIPSession(callID: callID)
} catch {
MXLog.error("Failed setting up VoIP session with error: \(error)")
}
}
}
override func process(viewAction: CallScreenViewAction) {
switch viewAction {
case .urlChanged(let url):
guard let url else { return }
MXLog.info("URL changed to: \(url)")
}
}
// MARK: - CXCallObserverDelegate
func callObserver(_ callObserver: CXCallObserver, callChanged call: CXCall) {
MXLog.info("Call changed: \(call)")
}
// MARK: - CXProviderDelegate
func providerDidReset(_ provider: CXProvider) {
MXLog.info("Call provider did reset: \(provider)")
}
// MARK: - Private
private static let eventHandlerName = "elementx"
private static var eventHandlerInjectionScript: String {
"""
window.addEventListener(
"message",
(event) => {
let message = {data: event.data, origin: event.origin}
if (message.data.response && message.data.api == "toWidget"
|| !message.data.response && message.data.api == "fromWidget") {
window.webkit.messageHandlers.\(eventHandlerName).postMessage(JSON.stringify(message.data));
}else{
console.log("-- skipped event handling by the client because it is send from the client itself.");
}
},
false,
);
"""
}
private func evaluateJavaScript(_ script: String) async -> String? {
guard let evaluator = state.bindings.javaScriptEvaluator else {
fatalError("Invalid javaScriptEvaluator")
}
do {
return try await evaluator(script) as? String
} catch {
MXLog.error("Failed evaluating javaScript with error: \(error)")
return nil
}
}
private func setupVoIPSession(callID: UUID) async throws {
try AVAudioSession.sharedInstance().setCategory(.playAndRecord, mode: .videoChat, options: [])
try AVAudioSession.sharedInstance().setActive(true, options: .notifyOthersOnDeactivation)
let handle = CXHandle(type: .generic, value: roomProxy.roomTitle)
let startCallAction = CXStartCallAction(call: callID, handle: handle)
startCallAction.isVideo = true
let transaction = CXTransaction(action: startCallAction)
try await callController.request(transaction)
}
private nonisolated func tearDownVoIPSession(callID: UUID?) {
guard let callID else {
return
}
try? AVAudioSession.sharedInstance().setActive(false)
let endCallAction = CXEndCallAction(call: callID)
let transaction = CXTransaction(action: endCallAction)
callController.request(transaction) { error in
if let error {
MXLog.error("Failed transaction with error: \(error)")
}
}
}
}
@@ -0,0 +1,23 @@
//
// Copyright 2022 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import Combine
@MainActor
protocol CallScreenViewModelProtocol {
var actions: AnyPublisher<CallScreenViewModelAction, Never> { get }
var context: CallScreenViewModelType.Context { get }
}
@@ -0,0 +1,181 @@
//
// Copyright 2022 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import Combine
import SwiftUI
import WebKit
struct CallScreen: View {
@ObservedObject var context: CallScreenViewModel.Context
var body: some View {
WebView(url: context.viewState.url, viewModelContext: context)
.navigationTitle("Call")
.navigationBarTitleDisplayMode(.inline)
.ignoresSafeArea(edges: .bottom)
.presentationDragIndicator(.visible)
.environment(\.colorScheme, .dark)
.alert(item: $context.alertInfo)
}
}
private struct WebView: UIViewRepresentable {
let url: URL?
let viewModelContext: CallScreenViewModel.Context
func makeUIView(context: Context) -> WKWebView {
context.coordinator.webView
}
func makeCoordinator() -> Coordinator {
Coordinator(viewModelContext: viewModelContext)
}
func updateUIView(_ webView: WKWebView, context: Context) {
if let url {
context.coordinator.url = url
let request = URLRequest(url: url)
webView.load(request)
}
}
@MainActor
class Coordinator: NSObject, WKScriptMessageHandler, WKUIDelegate, WKNavigationDelegate {
private let viewModelContext: CallScreenViewModel.Context
private var webViewURLObservation: NSKeyValueObservation?
private(set) var webView: WKWebView!
var url: URL!
init(viewModelContext: CallScreenViewModel.Context) {
self.viewModelContext = viewModelContext
super.init()
DispatchQueue.main.async {
// Avoid `Publishing changes from within view update warnings`
viewModelContext.javaScriptEvaluator = self.evaluateJavaScript(_:)
}
let configuration = WKWebViewConfiguration()
let userContentController = WKUserContentController()
userContentController.add(self, name: viewModelContext.viewState.messageHandler)
configuration.userContentController = userContentController
configuration.allowsInlineMediaPlayback = true
configuration.allowsPictureInPictureMediaPlayback = true
if let script = viewModelContext.viewState.script {
let userScript = WKUserScript(source: script, injectionTime: .atDocumentEnd, forMainFrameOnly: false)
configuration.userContentController.addUserScript(userScript)
}
webView = WKWebView(frame: .zero, configuration: configuration)
webView.uiDelegate = self
webView.navigationDelegate = self
// Try matching Element Call colors
webView.isOpaque = false
webView.backgroundColor = .compound.bgCanvasDefault
webView.scrollView.backgroundColor = .compound.bgCanvasDefault
}
func evaluateJavaScript(_ script: String) async throws -> Any? {
// After testing different scenarios it seems that when using async/await version of these
// methods wkwebView expects JavaScript to return with a value (something other than Void),
// if there is no value returning from the JavaScript that you evaluate you will have a crash.
try await withCheckedThrowingContinuation { continuaton in
webView.evaluateJavaScript(script) { result, error in
if let error {
continuaton.resume(throwing: error)
} else {
continuaton.resume(returning: result)
}
}
}
}
// MARK: - WKScriptMessageHandler
nonisolated func userContentController(_ userContentController: WKUserContentController,
didReceive message: WKScriptMessage) {
Task { @MainActor in
viewModelContext.javaScriptMessageHandler?(message.body)
}
}
// MARK: - WKUIDelegate
func webView(_ webView: WKWebView, decideMediaCapturePermissionsFor origin: WKSecurityOrigin, initiatedBy frame: WKFrameInfo, type: WKMediaCaptureType) async -> WKPermissionDecision {
// Don't allow permissions for domains different than what the call was started on
guard origin.host == url.host else {
return .deny
}
return .grant
}
// MARK: - WKNavigationDelegate
func webView(_ webView: WKWebView, decidePolicyFor navigationAction: WKNavigationAction) async -> WKNavigationActionPolicy {
// Allow any content from the main URL.
if navigationAction.request.url?.host == url.host {
return .allow
}
// Additionally allow any embedded content such as captchas.
if let targetFrame = navigationAction.targetFrame, !targetFrame.isMainFrame {
return .allow
}
// Otherwise the request is invalid.
return .cancel
}
nonisolated func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
Task { @MainActor in
viewModelContext.send(viewAction: .urlChanged(webView.url))
}
}
}
}
// MARK: - Previews
struct CallScreen_Previews: PreviewProvider {
static let viewModel = {
let roomProxy = RoomProxyMock()
let widgetDriver = ElementCallWidgetDriverMock()
widgetDriver.underlyingMessagePublisher = .init()
widgetDriver.underlyingActions = PassthroughSubject<ElementCallWidgetDriverAction, Never>().eraseToAnyPublisher()
widgetDriver.startBaseURLClientIDReturnValue = .success(URL.userDirectory)
roomProxy.elementCallWidgetDriverReturnValue = widgetDriver
return CallScreenViewModel(roomProxy: roomProxy,
callBaseURL: "https://call.element.io",
clientID: "io.element.elementx")
}()
static var previews: some View {
NavigationStack {
CallScreen(context: viewModel.context)
}
}
}
@@ -39,6 +39,7 @@ enum RoomScreenCoordinatorAction {
case presentEmojiPicker(itemID: TimelineItemIdentifier, selectedEmojis: Set<String>)
case presentRoomMemberDetails(member: RoomMemberProxyProtocol)
case presentMessageForwarding(itemID: TimelineItemIdentifier)
case presentCallScreen
}
final class RoomScreenCoordinator: CoordinatorProtocol {
@@ -109,6 +110,8 @@ final class RoomScreenCoordinator: CoordinatorProtocol {
actionsSubject.send(.presentLocationViewer(body: body, geoURI: geoURI, description: description))
case .composer(let action):
composerViewModel.process(roomAction: action)
case .displayCallScreen:
actionsSubject.send(.presentCallScreen)
}
}
.store(in: &cancellables)
@@ -34,6 +34,7 @@ enum RoomScreenViewModelAction {
case displayMessageForwarding(itemID: TimelineItemIdentifier)
case displayLocation(body: String, geoURI: GeoURI, description: String?)
case composer(action: RoomScreenComposerAction)
case displayCallScreen
}
enum RoomScreenComposerMode: Equatable {
@@ -63,7 +64,7 @@ enum RoomScreenViewAction {
case paginateBackwards
case selectedPollOption(pollStartID: String, optionID: String)
case endPoll(pollStartID: String)
case timelineItemMenu(itemID: TimelineItemIdentifier)
case timelineItemMenuAction(itemID: TimelineItemIdentifier, action: TimelineItemMenuAction)
@@ -73,7 +74,7 @@ enum RoomScreenViewAction {
case tappedOnUser(userID: String)
case reactionSummary(itemID: TimelineItemIdentifier, key: String)
case retrySend(itemID: TimelineItemIdentifier)
case cancelSend(itemID: TimelineItemIdentifier)
@@ -81,8 +82,11 @@ enum RoomScreenViewAction {
case enableLongPress(itemID: TimelineItemIdentifier)
case disableLongPress(itemID: TimelineItemIdentifier)
case playPauseAudio(itemID: TimelineItemIdentifier)
case seekAudio(itemID: TimelineItemIdentifier, progress: Double)
case presentCall
}
enum RoomScreenComposerAction {
@@ -103,9 +107,13 @@ struct RoomScreenViewState: BindableState {
var isEncryptedOneToOneRoom = false
var timelineViewState = TimelineViewState() // check the doc before changing this
var swiftUITimelineEnabled = false
var longPressDisabledItemID: TimelineItemIdentifier?
var ownUserID: String
var showCallButton = false
var isCallOngoing = false
var bindings: RoomScreenViewStateBindings
/// A closure providing the actions to show when long pressing on an item in the timeline.
@@ -127,14 +135,14 @@ struct RoomScreenViewStateBindings {
/// Information describing the currently displayed alert.
var alertInfo: AlertInfo<RoomScreenErrorType>?
/// An alert info for confirmation actions (e.g. ending a poll)
var confirmationAlertInfo: AlertInfo<UUID>?
var debugInfo: TimelineItemDebugInfo?
var actionMenuInfo: TimelineItemActionMenuInfo?
var sendFailedConfirmationDialogInfo: SendFailedConfirmationDialogInfo?
var reactionSummaryInfo: ReactionSummaryInfo?
@@ -144,7 +152,7 @@ struct TimelineItemActionMenuInfo: Equatable, Identifiable {
static func == (lhs: TimelineItemActionMenuInfo, rhs: TimelineItemActionMenuInfo) -> Bool {
lhs.id == rhs.id
}
let item: EventBasedTimelineItemProtocol
var id: TimelineItemIdentifier {
@@ -154,7 +162,7 @@ struct TimelineItemActionMenuInfo: Equatable, Identifiable {
struct SendFailedConfirmationDialogInfo: ConfirmationDialogProtocol {
let title = L10n.screenRoomRetrySendMenuTitle
let itemID: TimelineItemIdentifier
}
@@ -184,19 +192,19 @@ struct RoomMemberState {
struct TimelineViewState {
var canBackPaginate = true
var isBackPaginating = false
// These can be removed when we have full swiftUI and moved as @State values in the view
var scrollToBottomPublisher = PassthroughSubject<Void, Never>()
var itemsDictionary = OrderedDictionary<String, RoomTimelineItemViewState>()
var renderedTimelineIDs = [String]()
var pendingTimelineIDs = [String]()
var timelineIDs: [String] {
itemsDictionary.keys.elements
}
var itemViewStates: [RoomTimelineItemViewState] {
renderedTimelineIDs.compactMap { itemsDictionary[$0] }
}
@@ -63,6 +63,7 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
readReceiptsEnabled: appSettings.readReceiptsEnabled,
isEncryptedOneToOneRoom: roomProxy.isEncryptedOneToOneRoom,
ownUserID: roomProxy.ownUserID,
isCallOngoing: roomProxy.isCallOngoing,
bindings: .init(reactionsCollapsed: [:])),
imageProvider: mediaProvider)
@@ -163,6 +164,8 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
message: L10n.commonPollEndConfirmation,
primaryButton: .init(title: L10n.actionCancel, role: .cancel, action: nil),
secondaryButton: .init(title: L10n.actionOk, action: { self.endPoll(pollStartID: pollStartID) }))
case .presentCall:
actionsSubject.send(.displayCallScreen)
}
}
@@ -243,6 +246,10 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
.weakAssign(to: \.state.readReceiptsEnabled, on: self)
.store(in: &cancellables)
appSettings.$elementCallEnabled
.weakAssign(to: \.state.showCallButton, on: self)
.store(in: &cancellables)
roomProxy.members
.map { members in
members.reduce(into: [String: RoomMemberState]()) { dictionary, member in
@@ -14,6 +14,7 @@
// limitations under the License.
//
import Compound
import SwiftUI
import WysiwygComposer
@@ -155,8 +156,33 @@ struct RoomScreen: View {
ToolbarItem(placement: .principal) {
RoomHeaderView(context: context)
}
ToolbarItem(placement: .primaryAction) {
callButton
}
}
@ViewBuilder
private var callButton: some View {
if context.viewState.showCallButton {
if context.viewState.isCallOngoing {
Button {
context.send(viewAction: .presentCall)
} label: {
Label(L10n.actionJoin, icon: \.videoCall)
.labelStyle(.titleAndIcon)
}
.buttonStyle(ElementCallButtonStyle())
} else {
Button {
context.send(viewAction: .presentCall)
} label: {
CompoundIcon(\.videoCall)
}
}
}
}
private var isNavigationBarHidden: Bool {
composerToolbarContext.composerActionsEnabled && composerToolbarContext.composerExpanded && UIDevice.current.userInterfaceIdiom == .pad
}
@@ -167,11 +193,11 @@ struct RoomScreen: View {
struct RoomScreen_Previews: PreviewProvider, TestablePreview {
static let viewModel = RoomScreenViewModel(timelineController: MockRoomTimelineController(),
mediaProvider: MockMediaProvider(),
roomProxy: RoomProxyMock(with: .init(displayName: "Preview room")),
roomProxy: RoomProxyMock(with: .init(displayName: "Preview room", isCallOngoing: true)),
appSettings: ServiceLocator.shared.settings,
analytics: ServiceLocator.shared.analytics,
userIndicatorController: ServiceLocator.shared.userIndicatorController)
static var previews: some View {
NavigationStack {
RoomScreen(context: viewModel.context, composerToolbar: ComposerToolbar.mock())
@@ -52,6 +52,7 @@ protocol DeveloperOptionsProtocol: AnyObject {
var voiceMessageEnabled: Bool { get set }
var mentionsEnabled: Bool { get set }
var appLockFlowEnabled: Bool { get set }
var elementCallEnabled: Bool { get set }
}
extension AppSettings: DeveloperOptionsProtocol { }
@@ -50,6 +50,10 @@ struct DeveloperOptionsScreen: View {
Text("Show user mentions")
Text("Requires app reboot")
}
Toggle(isOn: $context.elementCallEnabled) {
Text("Elemement Call")
}
}
Section("Room creation") {