EC: Native switch for audio outputs/inputs and earpiece (#4136)

* EC: native switch for audio outputs/inputs and earpiece

also includes proximity sensor enabling when the earpiece is used.

* test disabling and re-enabling audio on audio change

* Revert "test disabling and re-enabling audio on audio change"

This reverts commit 907c791b8df7009214f5c164ecf8fe7d90a8addc.

* pr suggestions

* pr suggestion and small refactor of the widgetAction handler

ElementX/Sources/Screens/CallScreen/CallScreenModels.swift

@@ -16,7 +16,6 @@ enum CallScreenViewModelAction {
 }
 
 struct CallScreenViewState: BindableState {
-    let messageHandler: String
     let script: String?
     var url: URL?
     
@@ -26,7 +25,6 @@ struct CallScreenViewState: BindableState {
 }
 
 struct Bindings {
-    var javaScriptMessageHandler: ((Any) -> Void)?
     var javaScriptEvaluator: ((String) async throws -> Any)?
     var requestPictureInPictureHandler: (() async -> Result<Void, CallScreenError>)?
     
@@ -39,8 +37,60 @@ enum CallScreenViewAction {
     case navigateBack
     case pictureInPictureWillStop
     case endCall
+    case mediaCapturePermissionGranted
+    case outputDeviceSelected(deviceID: String)
+    case widgetAction(message: String)
 }
 
 enum CallScreenError: Error {
     case pictureInPictureNotAvailable
 }
+
+/// Identifies each event handler used by the CallScreen webview
+///
+/// The names of the enum need to always match the name of the handlers on the webview.
+enum CallScreenJavaScriptMessageName: String, CaseIterable {
+    /// Widget actions's handler.
+    case widgetAction
+    /// Used to show the native AVRoutePickerView.
+    case showNativeOutputDevicePicker
+    /// Used to determine if the webview has selected the earpiece or not.
+    case onOutputDeviceSelect
+    
+    private var postMessageScript: String {
+        switch self {
+        case .widgetAction:
+            """
+            window.addEventListener(
+                "message",
+                (event) => {
+                    let message = {data: event.data, origin: event.origin};
+                    if (message.data.response && message.data.api == "toWidget"
+                    || !message.data.response && message.data.api == "fromWidget") {
+                        window.webkit.messageHandlers.\(rawValue).postMessage(JSON.stringify(message.data));
+                    } else {
+                        console.log("-- skipped event handling by the client because it is send from the client itself.");
+                    }
+                },
+                false,
+            );
+            """
+        case .showNativeOutputDevicePicker:
+            """
+            window.controls.\(rawValue) = () => {
+                window.webkit.messageHandlers.\(rawValue).postMessage("");
+            };
+            """
+        case .onOutputDeviceSelect:
+            """
+            window.controls.\(rawValue) = (id) => {
+                window.webkit.messageHandlers.\(rawValue).postMessage(id);
+            };
+            """
+        }
+    }
+    
+    static var allCasesInjectionScript: String {
+        allCases.map(\.postMessageScript).joined(separator: "\n")
+    }
+}

ElementX/Sources/Screens/CallScreen/CallScreenViewModel.swift

@@ -52,15 +52,9 @@ class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol
             widgetDriver = roomProxy.elementCallWidgetDriver(deviceID: deviceID)
         }
         
-        super.init(initialViewState: CallScreenViewState(messageHandler: Self.eventHandlerName,
-                                                         script: Self.eventHandlerInjectionScript,
+        super.init(initialViewState: CallScreenViewState(script: CallScreenJavaScriptMessageName.allCasesInjectionScript,
                                                          certificateValidator: appHooks.certificateValidatorHook))
         
-        state.bindings.javaScriptMessageHandler = { [weak self] message in
-            guard let self, let message = message as? String else { return }
-            Task { await self.widgetDriver.handleMessage(message) }
-        }
-        
         elementCallService.actions
             .receive(on: DispatchQueue.main)
             .sink { [weak self] action in
@@ -107,6 +101,13 @@ class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol
             }
             .store(in: &cancellables)
         
+        NotificationCenter.default
+            .publisher(for: AVAudioSession.routeChangeNotification)
+            .sink { [weak self] _ in
+                Task { await self?.updateOutputsListOnWeb() }
+            }
+            .store(in: &cancellables)
+        
         setupCall()
     }
     
@@ -123,6 +124,12 @@ class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol
             actionsSubject.send(.pictureInPictureStopped)
         case .endCall:
             actionsSubject.send(.dismiss)
+        case .mediaCapturePermissionGranted:
+            Task { await updateOutputsListOnWeb() }
+        case .outputDeviceSelected(deviceID: let deviceID):
+            handleOutputDeviceSelected(deviceID: deviceID)
+        case .widgetAction(let message):
+            Task { await widgetDriver.handleMessage(message) }
         }
     }
     
@@ -132,6 +139,7 @@ class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol
         }
         
         elementCallService.tearDownCallSession()
+        UIDevice.current.isProximityMonitoringEnabled = false
     }
     
     // MARK: - Private
@@ -183,6 +191,15 @@ class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol
         }
     }
     
+    // This should always match the web app value
+    private static let earpieceID = "earpiece-id"
+    
+    private func handleOutputDeviceSelected(deviceID: String) {
+        let isEarpiece = deviceID == Self.earpieceID
+        MXLog.info("[handleOutputDeviceSelected] is earpiece: \(isEarpiece)")
+        UIDevice.current.isProximityMonitoringEnabled = isEarpiece
+    }
+    
     private func handleBackwardsNavigation() async {
         guard state.url != nil,
               isPictureInPictureAllowed,
@@ -242,23 +259,29 @@ class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol
         }
     }
     
-    private static let eventHandlerName = "elementx"
-    
-    private static var eventHandlerInjectionScript: String {
-        """
-        window.addEventListener(
-            "message",
-            (event) => {
-                let message = {data: event.data, origin: event.origin}
-                if (message.data.response && message.data.api == "toWidget"
-                || !message.data.response && message.data.api == "fromWidget") {
-                  window.webkit.messageHandlers.\(eventHandlerName).postMessage(JSON.stringify(message.data));
-                }else{
-                  console.log("-- skipped event handling by the client because it is send from the client itself.");
-                }
-            },
-            false,
-          );
-        """
+    /// This function updates the list of available audio outputs on the web side
+    /// however since we actually handle switching the audio output through the OS,
+    /// this is only used to inform the webview when the speaker is selected,
+    /// so that the option to use the earpiece can be displayed.
+    private func updateOutputsListOnWeb() async {
+        guard let currentOutput = AVAudioSession.sharedInstance().currentRoute.outputs.first else {
+            return
+        }
+        
+        let deviceList = if currentOutput.portType == .builtInSpeaker {
+            // This allows the webview to display the earpiece option
+            "{id: '\(currentOutput.uid)', name: '\(currentOutput.portName)', forEarpiece: true}"
+        } else {
+            // Doesn't matter because the switch is handled through the OS
+            "{id: 'dummy', name: 'dummy'}"
+        }
+        
+        let javaScript = "window.controls.setAvailableOutputDevices([\(deviceList)])"
+        do {
+            let result = try await state.bindings.javaScriptEvaluator?(javaScript)
+            MXLog.debug("Evaluated  with result: \(String(describing: result))")
+        } catch {
+            MXLog.error("Received javascript evaluation error: \(error)")
+        }
     }
 }

ElementX/Sources/Screens/CallScreen/View/CallScreen.swift

@@ -75,6 +75,7 @@ private struct CallView: UIViewRepresentable {
         private var webView: WKWebView!
         private var pictureInPictureController: AVPictureInPictureController?
         private let pictureInPictureViewController: AVPictureInPictureVideoCallViewController
+        private var routePickerView: AVRoutePickerView!
         
         /// The view to be shown in the app. This will contain the web view when picture in picture isn't running.
         let webViewWrapper = WebViewWrapper(frame: .zero)
@@ -97,7 +98,9 @@ private struct CallView: UIViewRepresentable {
             let configuration = WKWebViewConfiguration()
             
             let userContentController = WKUserContentController()
-            userContentController.add(WKScriptMessageHandlerWrapper(self), name: viewModelContext.viewState.messageHandler)
+            CallScreenJavaScriptMessageName.allCases.forEach {
+                userContentController.add(WKScriptMessageHandlerWrapper(self), name: $0.rawValue)
+            }
             
             // Required to allow a webview that uses file URL to load its own assets
             configuration.preferences.setValue(true, forKey: "allowFileAccessFromFileURLs")
@@ -123,6 +126,12 @@ private struct CallView: UIViewRepresentable {
             webView.backgroundColor = .compound.bgCanvasDefault
             webView.scrollView.backgroundColor = .compound.bgCanvasDefault
             
+            // This button is always hidden and is only used to be programmaticaly tapped
+            routePickerView = AVRoutePickerView(frame: .zero)
+            routePickerView.isHidden = true
+            routePickerView.isUserInteractionEnabled = false
+            webView.addSubview(routePickerView)
+            
             webViewWrapper.addMatchedSubview(webView)
             
             if AVPictureInPictureController.isPictureInPictureSupported() {
@@ -161,7 +170,32 @@ private struct CallView: UIViewRepresentable {
         }
         
         func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) {
-            viewModelContext?.javaScriptMessageHandler?(message.body)
+            guard let handlerID = CallScreenJavaScriptMessageName(rawValue: message.name) else {
+                return
+            }
+            
+            switch handlerID {
+            case .widgetAction:
+                guard let message = message.body as? String else { return }
+                viewModelContext?.send(viewAction: .widgetAction(message: message))
+            case .showNativeOutputDevicePicker:
+                DispatchQueue.main.async {
+                    self.tapRoutePickerView()
+                }
+            case .onOutputDeviceSelect:
+                guard let deviceID = message.body as? String else { return }
+                viewModelContext?.send(viewAction: .outputDeviceSelected(deviceID: deviceID))
+            }
+        }
+        
+        // This function is called by the webview output routing button
+        // it allows to open the OS output selector using the hidden button.
+        private func tapRoutePickerView() {
+            guard let button = routePickerView.subviews.first(where: { $0 is UIButton }) as? UIButton else {
+                return
+            }
+            
+            button.sendActions(for: .touchUpInside)
         }
         
         // MARK: - WKUIDelegate
@@ -172,6 +206,7 @@ private struct CallView: UIViewRepresentable {
                 return .deny
             }
             
+            viewModelContext?.send(viewAction: .mediaCapturePermissionGranted)
             return .grant
         }
         

ElementX/Sources/Services/ElementCall/ElementCallWidgetDriver.swift

@@ -97,8 +97,7 @@ class ElementCallWidgetDriver: WidgetCapabilitiesProvider, ElementCallWidgetDriv
                                                                           rageshakeSubmitUrl: rageshakeURL,
                                                                           sentryDsn: analyticsConfiguration?.sentryDSN,
                                                                           sentryEnvironment: nil,
-                                                                          // Set this to false until we have the full implementation otherwise should be false only on macOS
-                                                                          controlledMediaDevices: false))
+                                                                          controlledMediaDevices: !ProcessInfo.processInfo.isiOSAppOnMac))
         } catch {
             MXLog.error("Failed to build widget settings: \(error)")
             return .failure(.failedBuildingWidgetSettings)