Implement support for ElementCall through Rust side widgets (#1906)

* Implement support for ElementCall through Rust side widgets * Add Element Call feature flag * Fix the unit tests * Address PR comments
2026-11-03 06:55:59 +00:00 · 2023-10-17 12:41:56 +03:00
parent 524e4b3f1e
commit 6e8dff2ef6
25 changed files with 1059 additions and 18 deletions
@@ -0,0 +1,65 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+import SwiftUI
+
+struct CallScreenCoordinatorParameters {
+    let roomProxy: RoomProxyProtocol
+    /// Which Element Call instance should be used
+    let callBaseURL: URL
+    /// A way to identify the current client against Element Call
+    let clientID: String
+}
+
+enum CallScreenCoordinatorAction {
+    case dismiss
+}
+
+final class CallScreenCoordinator: CoordinatorProtocol {
+    private let parameters: CallScreenCoordinatorParameters
+    private var viewModel: CallScreenViewModelProtocol
+    private let actionsSubject: PassthroughSubject<CallScreenCoordinatorAction, Never> = .init()
+    
+    private var cancellables: Set<AnyCancellable> = .init()
+    var actions: AnyPublisher<CallScreenCoordinatorAction, Never> {
+        actionsSubject.eraseToAnyPublisher()
+    }
+    
+    init(parameters: CallScreenCoordinatorParameters) {
+        self.parameters = parameters
+        
+        viewModel = CallScreenViewModel(roomProxy: parameters.roomProxy,
+                                        callBaseURL: parameters.callBaseURL,
+                                        clientID: parameters.clientID)
+    }
+    
+    func start() {
+        viewModel.actions.sink { [weak self] action in
+            guard let self else { return }
+            
+            switch action {
+            case .dismiss:
+                actionsSubject.send(.dismiss)
+            }
+        }
+        .store(in: &cancellables)
+    }
+        
+    func toPresentable() -> AnyView {
+        AnyView(CallScreen(context: viewModel.context))
+    }
+}
@@ -0,0 +1,39 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+
+enum CallScreenViewModelAction {
+    case dismiss
+}
+
+struct CallScreenViewState: BindableState {
+    let messageHandler: String
+    let script: String?
+    var url: URL?
+    var bindings = Bindings()
+}
+
+struct Bindings {
+    var javaScriptMessageHandler: ((Any) -> Void)?
+    var javaScriptEvaluator: ((String) async throws -> Any)?
+    
+    var alertInfo: AlertInfo<UUID>?
+}
+
+enum CallScreenViewAction {
+    case urlChanged(URL?)
+}
@@ -0,0 +1,206 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import AVFoundation
+import CallKit
+import Combine
+import SwiftUI
+
+typealias CallScreenViewModelType = StateStoreViewModel<CallScreenViewState, CallScreenViewAction>
+
+class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol {
+    private let roomProxy: RoomProxyProtocol
+    private let callBaseURL: URL
+    private let clientID: String
+    
+    private let widgetDriver: ElementCallWidgetDriverProtocol
+    
+    private let callController = CXCallController()
+    private let callProvider = CXProvider(configuration: .init())
+    private let callID = UUID()
+    
+    private let actionsSubject: PassthroughSubject<CallScreenViewModelAction, Never> = .init()
+    var actions: AnyPublisher<CallScreenViewModelAction, Never> {
+        actionsSubject.eraseToAnyPublisher()
+    }
+    
+    deinit {
+        tearDownVoIPSession(callID: callID)
+    }
+    
+    /// Designated initialiser
+    /// - Parameters:
+    ///   - roomProxy: The room in which the call should be created
+    ///   - callBaseURL: Which Element Call instance should be used
+    ///   - clientID: Something to identify the current client on the Element Call side
+    init(roomProxy: RoomProxyProtocol, callBaseURL: URL, clientID: String) {
+        self.roomProxy = roomProxy
+        self.callBaseURL = callBaseURL
+        self.clientID = clientID
+        
+        widgetDriver = roomProxy.elementCallWidgetDriver()
+        
+        super.init(initialViewState: CallScreenViewState(messageHandler: Self.eventHandlerName,
+                                                         script: Self.eventHandlerInjectionScript))
+        
+        state.bindings.javaScriptMessageHandler = { [weak self] message in
+            guard let self,
+                  let message = message as? String else {
+                return
+            }
+            
+            Task {
+                await self.widgetDriver.sendMessage(message)
+            }
+        }
+        
+        widgetDriver.messagePublisher
+            .receive(on: DispatchQueue.main)
+            .sink { [weak self] receivedMessage in
+                guard let self else { return }
+                
+                Task {
+                    do {
+                        let message = "postMessage(\(receivedMessage), '*')"
+                        let result = try await self.state.bindings.javaScriptEvaluator?(message)
+                        MXLog.debug("Evaluated javascript: \(message) with result: \(String(describing: result))")
+                    } catch {
+                        MXLog.error("Received javascript evaluation error: \(error)")
+                    }
+                }
+            }
+            .store(in: &cancellables)
+        
+        widgetDriver.actions
+            .receive(on: DispatchQueue.main)
+            .sink { [weak self] action in
+                guard let self else { return }
+                
+                switch action {
+                case .callEnded:
+                    actionsSubject.send(.dismiss)
+                default:
+                    break
+                }
+            }
+            .store(in: &cancellables)
+        
+        Task {
+            switch await widgetDriver.start(baseURL: callBaseURL, clientID: clientID) {
+            case .success(let url):
+                state.url = url
+            case .failure(let error):
+                MXLog.error("Failed starting ElementCall Widget Driver with error: \(error)")
+                state.bindings.alertInfo = .init(id: UUID(), title: L10n.errorUnknown, primaryButton: .init(title: L10n.actionOk, action: { [weak self] in
+                    self?.actionsSubject.send(.dismiss)
+                }))
+                
+                return
+            }
+            
+            do {
+                try await setupVoIPSession(callID: callID)
+            } catch {
+                MXLog.error("Failed setting up VoIP session with error: \(error)")
+            }
+        }
+    }
+    
+    override func process(viewAction: CallScreenViewAction) {
+        switch viewAction {
+        case .urlChanged(let url):
+            guard let url else { return }
+            MXLog.info("URL changed to: \(url)")
+        }
+    }
+    
+    // MARK: - CXCallObserverDelegate
+    
+    func callObserver(_ callObserver: CXCallObserver, callChanged call: CXCall) {
+        MXLog.info("Call changed: \(call)")
+    }
+    
+    // MARK: - CXProviderDelegate
+    
+    func providerDidReset(_ provider: CXProvider) {
+        MXLog.info("Call provider did reset: \(provider)")
+    }
+    
+    // MARK: - Private
+
+    private static let eventHandlerName = "elementx"
+    
+    private static var eventHandlerInjectionScript: String {
+        """
+        window.addEventListener(
+            "message",
+            (event) => {
+                let message = {data: event.data, origin: event.origin}
+                if (message.data.response && message.data.api == "toWidget"
+                || !message.data.response && message.data.api == "fromWidget") {
+                  window.webkit.messageHandlers.\(eventHandlerName).postMessage(JSON.stringify(message.data));
+                }else{
+                  console.log("-- skipped event handling by the client because it is send from the client itself.");
+                }
+            },
+            false,
+          );
+        """
+    }
+    
+    private func evaluateJavaScript(_ script: String) async -> String? {
+        guard let evaluator = state.bindings.javaScriptEvaluator else {
+            fatalError("Invalid javaScriptEvaluator")
+        }
+        
+        do {
+            return try await evaluator(script) as? String
+        } catch {
+            MXLog.error("Failed evaluating javaScript with error: \(error)")
+            return nil
+        }
+    }
+    
+    private func setupVoIPSession(callID: UUID) async throws {
+        try AVAudioSession.sharedInstance().setCategory(.playAndRecord, mode: .videoChat, options: [])
+        try AVAudioSession.sharedInstance().setActive(true, options: .notifyOthersOnDeactivation)
+        
+        let handle = CXHandle(type: .generic, value: roomProxy.roomTitle)
+        let startCallAction = CXStartCallAction(call: callID, handle: handle)
+        startCallAction.isVideo = true
+        
+        let transaction = CXTransaction(action: startCallAction)
+        
+        try await callController.request(transaction)
+    }
+    
+    private nonisolated func tearDownVoIPSession(callID: UUID?) {
+        guard let callID else {
+            return
+        }
+        
+        try? AVAudioSession.sharedInstance().setActive(false)
+            
+        let endCallAction = CXEndCallAction(call: callID)
+        let transaction = CXTransaction(action: endCallAction)
+        
+        callController.request(transaction) { error in
+            if let error {
+                MXLog.error("Failed transaction with error: \(error)")
+            }
+        }
+    }
+}
@@ -0,0 +1,23 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+
+@MainActor
+protocol CallScreenViewModelProtocol {
+    var actions: AnyPublisher<CallScreenViewModelAction, Never> { get }
+    var context: CallScreenViewModelType.Context { get }
+}
@@ -0,0 +1,181 @@
+//
+// Copyright 2022 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Combine
+import SwiftUI
+import WebKit
+
+struct CallScreen: View {
+    @ObservedObject var context: CallScreenViewModel.Context
+    
+    var body: some View {
+        WebView(url: context.viewState.url, viewModelContext: context)
+            .navigationTitle("Call")
+            .navigationBarTitleDisplayMode(.inline)
+            .ignoresSafeArea(edges: .bottom)
+            .presentationDragIndicator(.visible)
+            .environment(\.colorScheme, .dark)
+            .alert(item: $context.alertInfo)
+    }
+}
+
+private struct WebView: UIViewRepresentable {
+    let url: URL?
+    let viewModelContext: CallScreenViewModel.Context
+    
+    func makeUIView(context: Context) -> WKWebView {
+        context.coordinator.webView
+    }
+    
+    func makeCoordinator() -> Coordinator {
+        Coordinator(viewModelContext: viewModelContext)
+    }
+    
+    func updateUIView(_ webView: WKWebView, context: Context) {
+        if let url {
+            context.coordinator.url = url
+            let request = URLRequest(url: url)
+            webView.load(request)
+        }
+    }
+    
+    @MainActor
+    class Coordinator: NSObject, WKScriptMessageHandler, WKUIDelegate, WKNavigationDelegate {
+        private let viewModelContext: CallScreenViewModel.Context
+        private var webViewURLObservation: NSKeyValueObservation?
+        
+        private(set) var webView: WKWebView!
+        
+        var url: URL!
+        
+        init(viewModelContext: CallScreenViewModel.Context) {
+            self.viewModelContext = viewModelContext
+            
+            super.init()
+            
+            DispatchQueue.main.async {
+                // Avoid `Publishing changes from within view update warnings`
+                viewModelContext.javaScriptEvaluator = self.evaluateJavaScript(_:)
+            }
+            
+            let configuration = WKWebViewConfiguration()
+            
+            let userContentController = WKUserContentController()
+            userContentController.add(self, name: viewModelContext.viewState.messageHandler)
+            
+            configuration.userContentController = userContentController
+            configuration.allowsInlineMediaPlayback = true
+            configuration.allowsPictureInPictureMediaPlayback = true
+            
+            if let script = viewModelContext.viewState.script {
+                let userScript = WKUserScript(source: script, injectionTime: .atDocumentEnd, forMainFrameOnly: false)
+                configuration.userContentController.addUserScript(userScript)
+            }
+            
+            webView = WKWebView(frame: .zero, configuration: configuration)
+            webView.uiDelegate = self
+            webView.navigationDelegate = self
+            
+            // Try matching Element Call colors
+            webView.isOpaque = false
+            webView.backgroundColor = .compound.bgCanvasDefault
+            webView.scrollView.backgroundColor = .compound.bgCanvasDefault
+        }
+        
+        func evaluateJavaScript(_ script: String) async throws -> Any? {
+            // After testing different scenarios it seems that when using async/await version of these
+            // methods wkwebView expects JavaScript to return with a value (something other than Void),
+            // if there is no value returning from the JavaScript that you evaluate you will have a crash.
+            try await withCheckedThrowingContinuation { continuaton in
+                webView.evaluateJavaScript(script) { result, error in
+                    if let error {
+                        continuaton.resume(throwing: error)
+                    } else {
+                        continuaton.resume(returning: result)
+                    }
+                }
+            }
+        }
+        
+        // MARK: - WKScriptMessageHandler
+        
+        nonisolated func userContentController(_ userContentController: WKUserContentController,
+                                               didReceive message: WKScriptMessage) {
+            Task { @MainActor in
+                viewModelContext.javaScriptMessageHandler?(message.body)
+            }
+        }
+        
+        // MARK: - WKUIDelegate
+        
+        func webView(_ webView: WKWebView, decideMediaCapturePermissionsFor origin: WKSecurityOrigin, initiatedBy frame: WKFrameInfo, type: WKMediaCaptureType) async -> WKPermissionDecision {
+            // Don't allow permissions for domains different than what the call was started on
+            guard origin.host == url.host else {
+                return .deny
+            }
+            
+            return .grant
+        }
+        
+        // MARK: - WKNavigationDelegate
+        
+        func webView(_ webView: WKWebView, decidePolicyFor navigationAction: WKNavigationAction) async -> WKNavigationActionPolicy {
+            // Allow any content from the main URL.
+            if navigationAction.request.url?.host == url.host {
+                return .allow
+            }
+            
+            // Additionally allow any embedded content such as captchas.
+            if let targetFrame = navigationAction.targetFrame, !targetFrame.isMainFrame {
+                return .allow
+            }
+            
+            // Otherwise the request is invalid.
+            return .cancel
+        }
+        
+        nonisolated func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
+            Task { @MainActor in
+                viewModelContext.send(viewAction: .urlChanged(webView.url))
+            }
+        }
+    }
+}
+
+// MARK: - Previews
+
+struct CallScreen_Previews: PreviewProvider {
+    static let viewModel = {
+        let roomProxy = RoomProxyMock()
+        
+        let widgetDriver = ElementCallWidgetDriverMock()
+        widgetDriver.underlyingMessagePublisher = .init()
+        widgetDriver.underlyingActions = PassthroughSubject<ElementCallWidgetDriverAction, Never>().eraseToAnyPublisher()
+        widgetDriver.startBaseURLClientIDReturnValue = .success(URL.userDirectory)
+        
+        roomProxy.elementCallWidgetDriverReturnValue = widgetDriver
+        
+        return CallScreenViewModel(roomProxy: roomProxy,
+                                   callBaseURL: "https://call.element.io",
+                                   clientID: "io.element.elementx")
+    }()
+    
+    static var previews: some View {
+        NavigationStack {
+            CallScreen(context: viewModel.context)
+        }
+    }
+}