From f7d0a0576366553f5f14218ef14582629127839c Mon Sep 17 00:00:00 2001 From: Sedat Qaja Date: Mon, 19 Oct 2026 09:18:07 +0200 Subject: [PATCH] fix(healthchat): rewire the NSE and call service to upstream's settings --- .../ElementCall/ElementCallService.swift | 323 +++++++++++++++--- NSE/Sources/NotificationHandler.swift | 6 +- 2 files changed, 287 insertions(+), 42 deletions(-) diff --git a/ElementX/Sources/Services/ElementCall/ElementCallService.swift b/ElementX/Sources/Services/ElementCall/ElementCallService.swift index 3304baee9..d7838b711 100644 --- a/ElementX/Sources/Services/ElementCall/ElementCallService.swift +++ b/ElementX/Sources/Services/ElementCall/ElementCallService.swift @@ -9,6 +9,7 @@ import AVFoundation import CallKit import Combine +import ElementCall import Foundation import MatrixRustSDK import PushKit @@ -26,13 +27,14 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe let roomID: String let rtcNotificationID: String? let isVoiceCall: Bool + var isNative = false } + private let appSettings: AppSettings private let pushRegistry: PKPushRegistry - private let callController = CXCallController() + private let callController: CXCallControllerProtocol private let callProvider: CXProviderProtocol private let timeProvider: TimeProvider - private let appSettings: AppSettings private weak var clientProxy: ClientProxyProtocol? { didSet { @@ -52,6 +54,10 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe private var endUnansweredCallTask: Task? + /// A call answered for the native stack, until the stack takes it over. + private var answeredNativeCallID: CallID? + private var endPendingCallTask: Task? + private var ongoingCallID: CallID? { didSet { ongoingCallRoomIDSubject.send(ongoingCallID?.roomID) } } @@ -68,11 +74,26 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe private var declineListenerHandle: TaskHandle? - init(callProvider: CXProviderProtocol? = nil, timeProvider: TimeProvider? = nil, appSettings: AppSettings) { + /// The native call stack, for as long as there's a session running calls through it. + private var nativeCallStack: NativeCallStack? + private var nativeCallCancellable: AnyCancellable? + private var mediaProvider: MediaProviderProtocol? + + var nativeCallController: ElementCallController? { + nativeCallStack?.controller + } + + private var cancellables = Set() + + init(appSettings: AppSettings, + callProvider: CXProviderProtocol? = nil, + callController: CXCallControllerProtocol? = nil, + timeProvider: TimeProvider? = nil) { + self.appSettings = appSettings + self.callController = callController ?? CXCallController() pushRegistry = PKPushRegistry(queue: nil) self.timeProvider = timeProvider ?? TimeProvider(clock: ContinuousClock(), now: Date.init) - self.appSettings = appSettings if let callProvider { self.callProvider = callProvider @@ -97,45 +118,145 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe pushRegistry.desiredPushTypes = [.voIP] self.callProvider.setDelegate(self, queue: nil) + + appSettings.nativeCallEnabledPublisher + .sink { [weak self] _ in + self?.updateNativeCallStack() + } + .store(in: &cancellables) } - func setClientProxy(_ clientProxy: any ClientProxyProtocol) { - self.clientProxy = clientProxy + func setUserSession(_ userSession: UserSessionProtocol?) { + // A stack is built against one client proxy and can't be handed to another, which is what a + // soft logout or a cache clear brings: a new session without the sign out that drops this one. + if userSession?.clientProxy !== clientProxy { + stopNativeCallStack() + } + + clientProxy = userSession?.clientProxy + mediaProvider = userSession?.mediaProvider + + updateNativeCallStack() } - func setupCallSession(roomID: String, roomDisplayName: String) async { + @discardableResult + func handleNativeCallRequest(roomProxy: JoinedRoomProxyProtocol, isVoiceCall: Bool) -> Bool { + guard let nativeCallStack else { + return false + } + + nativeCallStack.handleCallRequest(roomProxy: roomProxy, isVoiceCall: isVoiceCall) + return true + } + + func minimizeNativeCall() { + nativeCallStack?.minimize() + } + + func restoreNativeCall() { + nativeCallStack?.restore() + } + + func setupCallSession(roomID: String, roomDisplayName: String, isVideo: Bool) async { + // Unlike the push handling below, this always runs with a session, so the stack itself is + // the answer rather than the setting that would have built one. + let isNative = nativeCallStack != nil + MXLog.info("Setting up a call session, native: \(isNative), video: \(isVideo), replacing an ongoing call: \(ongoingCallID != nil)") + // Drop any ongoing calls when starting a new one if ongoingCallID != nil { - tearDownCallSession() + tearDownCallSession(sendEndCallAction: true) } // If this starting from a ring reuse those identifiers // Make sure the roomID matches - let callID = if let incomingCallID, incomingCallID.roomID == roomID { + var callID = if let incomingCallID, incomingCallID.roomID == roomID { incomingCallID } else { - CallID(callKitID: UUID(), roomID: roomID, rtcNotificationID: nil, isVoiceCall: false) + CallID(callKitID: UUID(), roomID: roomID, rtcNotificationID: nil, isVoiceCall: !isVideo) } + callID.isNative = isNative + let isAnsweringIncomingCall = callID.rtcNotificationID != nil clearIncomingCallState() ongoingCallID = callID - // Don't bother starting another CallKit session as it won't work properly + // A web view call must not be tracked by CallKit, as that gives this process exclusive + // access to media and the web view runs in another (see `CXAnswerCallAction`). // https://developer.apple.com/forums//thread/767949?answerId=812951022#812951022 + guard isNative else { + return + } - // let handle = CXHandle(type: .generic, value: roomDisplayName) - // let startCallAction = CXStartCallAction(call: callID.callKitID, handle: handle) - // startCallAction.isVideo = true + // Answering the ringing call: it is already reported, keep it. + guard !isAnsweringIncomingCall else { + return + } - // do { - // try await callController.request(CXTransaction(action: startCallAction)) - // } catch { - // MXLog.error("Failed requesting start call action with error: \(error)") - // } + let handle = CXHandle(type: .generic, value: roomID) + let startCallAction = CXStartCallAction(call: callID.callKitID, handle: handle) + startCallAction.contactIdentifier = roomDisplayName + startCallAction.isVideo = isVideo + + // Awaited because our membership goes out straight after: a peer that sees it before the + // system knows about the call reads the call as answered elsewhere. + await requestTransaction(CXTransaction(action: startCallAction), describedAs: "start call") + + // Without this the system call UI shows the handle, i.e. the room ID. + let update = CXCallUpdate() + update.localizedCallerName = roomDisplayName + update.remoteHandle = handle + update.hasVideo = isVideo + callProvider.reportCall(with: callID.callKitID, updated: update) } - func tearDownCallSession() { - tearDownCallSession(sendEndCallAction: true) + func reportCallSessionConnected(roomID: String) { + guard let ongoingCallID, ongoingCallID.roomID == roomID else { + // The system ends an outgoing call that never reports connecting, so a silent guard + // here shows up much later as a call that hangs itself up. + MXLog.warning("Not reporting the call as connected, no matching ongoing call") + return + } + + guard ongoingCallID.isNative else { + MXLog.info("Not reporting a web view call as connected, CallKit doesn't track it") + return + } + + MXLog.info("Reporting the call as connected") + callProvider.reportOutgoingCall(with: ongoingCallID.callKitID, connectedAt: nil) + } + + func tearDownCallSession(roomID: String) { + if ongoingCallID?.roomID == roomID { + tearDownCallSession(sendEndCallAction: true) + return + } + + // Answered and handed over, but the stack couldn't take it. Ending it here rather than + // waiting for the watchdog takes the system call UI away straight away. + if answeredNativeCallID?.roomID == roomID { + MXLog.warning("Ending an answered call the native call stack didn't take over") + endPendingCall() + return + } + + MXLog.info("Not tearing down the call session, no call for room \(roomID)") + } + + func reportCallSessionEnded(roomID: String, reason: CXCallEndedReason) { + guard let ongoingCallID, ongoingCallID.roomID == roomID else { + MXLog.info("Not reporting the end of a call for \(roomID), it isn't the ongoing call") + return + } + + // A web view call is not tracked by CallKit, as in `reportCallSessionConnected`. + if ongoingCallID.isNative { + MXLog.info("Reporting the call as ended without us: \(reason.rawValue)") + callProvider.reportCall(with: ongoingCallID.callKitID, endedAt: nil, reason: reason) + } + + self.ongoingCallID = nil } func setAudioEnabled(_ enabled: Bool, roomID: String) { @@ -157,6 +278,57 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe } } + // MARK: - Native calls + + /// Builds or releases the call stack to match the session and the setting. + private func updateNativeCallStack() { + guard appSettings.nativeCallEnabled, let clientProxy else { + stopNativeCallStack() + return + } + + guard nativeCallStack == nil else { return } + + guard let transport = clientProxy.makeNativeCallTransport() else { + MXLog.error("Cannot start the native call stack without a transport") + return + } + + let style = ElementCallStyle(theme: NativeCallTheme(), + icons: NativeCallIcons(), + avatars: NativeCallAvatars(mediaProvider: mediaProvider), + strings: .init(you: L10n.commonYou, + error: L10n.commonError, + stop: L10n.actionStop, + back: L10n.actionBack)) + + // The stats overlay is raw RTP counters, so it follows whatever already reveals developer + // surface. Screen sharing stays off until the broadcast extension that makes it work lands. + let options = ElementCallOptions(isDeveloperModeEnabled: appSettings.developerOptionsEnabled) + + let stack = NativeCallStack(transport: transport, + system: NativeCallSystem(service: self), + options: options, + style: style) + nativeCallStack = stack + + nativeCallCancellable = stack.actions + .sink { [weak self] presentation in + self?.actionsSubject.send(.nativeCall(presentation)) + } + + Task { await stack.start() } + } + + private func stopNativeCallStack() { + guard let nativeCallStack else { return } + self.nativeCallStack = nil + nativeCallCancellable = nil + + nativeCallStack.stop() + actionsSubject.send(.nativeCall(.dismiss)) + } + // MARK: - PKPushRegistryDelegate func pushRegistry(_ registry: PKPushRegistry, didUpdate pushCredentials: PKPushCredentials, for type: PKPushType) { @@ -226,7 +398,7 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe // If not for audio call the app will not be put to foreground and the webview won't be able to handle the call... // Consequence: The call will be presented to the user as a video call in CallKit UI, // but once Element Call is launched it will correctly route to a voice-only call. - update.hasVideo = true + update.hasVideo = appSettings.nativeCallEnabled ? !isVoiceCall : true update.localizedCallerName = roomDisplayName // https://stackoverflow.com/a/41230020/730924 update.remoteHandle = .init(type: .generic, value: roomID) @@ -264,10 +436,19 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe func provider(_ provider: CXProvider, didActivate audioSession: AVAudioSession) { MXLog.info("Call provider did activate audio session") + actionsSubject.send(.audioSessionActivated) } func provider(_ provider: CXProvider, didDeactivate audioSession: AVAudioSession) { MXLog.info("Call provider did deactivate audio session") + actionsSubject.send(.audioSessionDeactivated) + } + + func provider(_ provider: CXProvider, perform action: CXStartCallAction) { + // The native controller configured the audio session already; CallKit activates it and + // reports back through `didActivate`. + action.fulfill() + callProvider.reportOutgoingCall(with: action.callUUID, startedConnectingAt: nil) } func providerDidReset(_ provider: CXProvider) { @@ -298,6 +479,15 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe // First fullfill the action action.fulfill() + if appSettings.nativeCallEnabled { + // The native stack owns media in this process, so the CallKit call stays up: the + // controller attaches to it in `setupCallSession`. + endUnansweredCallTask?.cancel() + endCallIfLeftPending(incomingCallID) + actionsSubject.send(.startCall(roomID: incomingCallID.roomID, isVoiceCall: incomingCallID.isVoiceCall)) + return + } + // And delay ending the call so that the app has enough time // to get deeplinked into DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { @@ -313,7 +503,7 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe if let ongoingCallID { actionsSubject.send(.setAudioEnabled(!action.isMuted, roomID: ongoingCallID.roomID)) } else { - MXLog.error("Failed muting/unmuting call, missing ongoingCallID") + MXLog.info("Ignoring a mute action without an ongoing call") } action.fulfill() @@ -324,30 +514,56 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe // This gets called for no reason on simulators, where CallKit // isn't even supported, ignore it. #else - if let ongoingCallID { - actionsSubject.send(.endCall(roomID: ongoingCallID.roomID)) - } - - if let incomingCallID { - Task { - await sendDeclineCallEvent(incomingCallID) - } - } - - if incomingCallID?.callKitID == action.callUUID { - clearIncomingCallState() - } - - tearDownCallSession(sendEndCallAction: false) - + endCall(withCallKitID: action.callUUID) action.fulfill() #endif } + /// Ends the call the system identified, whether that's the ongoing one, the ringing one, or + /// neither because it has already been replaced. + /// + /// Kept out of the delegate method so that it's reachable on the simulator, where the provider + /// performs end call actions of its own and the delegate has to ignore them. + func endCall(withCallKitID callKitID: UUID) { + MXLog.info("Ending the call for \(callKitID), ongoing: \(ongoingCallID != nil), incoming: \(incomingCallID != nil)") + + // Both branches are gated on the identifier. Leaving a call to start one in another room + // requests an end call action for the old call and replaces `ongoingCallID` before the + // system gets round to performing it, so an ungated handler reads it as the new call ending + // and hangs up the call that was just started. + if let ongoingCallID, ongoingCallID.callKitID == callKitID { + actionsSubject.send(.endCall(roomID: ongoingCallID.roomID)) + tearDownCallSession(sendEndCallAction: false) + } + + if let incomingCallID, incomingCallID.callKitID == callKitID { + Task { + await sendDeclineCallEvent(incomingCallID) + } + + clearIncomingCallState() + } + } + // MARK: - Private + /// Bridges the controller's completion handler, which is all ``CXCallControllerProtocol`` can + /// carry, back to an awaitable call. + private func requestTransaction(_ transaction: CXTransaction, describedAs description: String) async { + await withCheckedContinuation { continuation in + callController.request(transaction) { error in + if let error { + MXLog.error("Failed requesting \(description) action with error: \(error)") + } + + continuation.resume() + } + } + } + private func tearDownCallSession(sendEndCallAction: Bool = true) { if sendEndCallAction, let ongoingCallID { + MXLog.info("Requesting an end call action for the ongoing call") let transaction = CXTransaction(action: CXEndCallAction(call: ongoingCallID.callKitID)) callController.request(transaction) { error in if let error { @@ -405,6 +621,32 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe } } + /// Ends an answered call that the native stack never took over, which nothing else would: the + /// call is deliberately left up for the controller to attach to in `setupCallSession`. + private func endCallIfLeftPending(_ callID: CallID) { + answeredNativeCallID = callID + + endPendingCallTask = Task { [weak self] in + try? await self?.timeProvider.clock.sleep(for: .seconds(30)) + + guard let self, !Task.isCancelled, answeredNativeCallID == callID else { + return + } + + MXLog.error("The native call stack never took over the answered call, ending it") + endPendingCall() + } + } + + private func endPendingCall() { + guard let answeredNativeCallID else { + return + } + + callProvider.reportCall(with: answeredNativeCallID.callKitID, endedAt: nil, reason: .failed) + clearIncomingCallState() + } + private func sendDeclineCallEvent(_ incomingCallID: CallID) async { guard let rtcNotificationID = incomingCallID.rtcNotificationID else { MXLog.info("No rtc notification event to decline.") @@ -506,6 +748,9 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe private func clearIncomingCallState() { endUnansweredCallTask?.cancel() endUnansweredCallTask = nil + endPendingCallTask?.cancel() + endPendingCallTask = nil + answeredNativeCallID = nil declineListenerHandle?.cancel() declineListenerHandle = nil incomingCallRoomInfoCancellable = nil diff --git a/NSE/Sources/NotificationHandler.swift b/NSE/Sources/NotificationHandler.swift index 85762bed0..4d7412f77 100644 --- a/NSE/Sources/NotificationHandler.swift +++ b/NSE/Sources/NotificationHandler.swift @@ -273,8 +273,8 @@ nonisolated class NotificationHandler { /// Asks the push gateway to deliver a PushKit push with the call details, the way the legacy /// element-ios app did. The app's `PKPushRegistryDelegate` then reports the call to CallKit. private func requestVoIPPush(roomID: String, roomDisplayName: String, rtcNotifyEventID: String, expirationDate: Date, isVoiceCall: Bool) async { - guard let token = settings.pushKitDeviceToken, - let notifyURL = settings.voipPushGatewayNotifyURL.flatMap(URL.init(string:)) else { + guard let token = userSettings.app.pushKitDeviceToken, + let notifyURL = userSettings.app.voipPushGatewayNotifyURL.flatMap(URL.init(string:)) else { MXLog.warning("No PushKit token or push gateway stored by the app, cannot request a VoIP push") return } @@ -288,7 +288,7 @@ nonisolated class NotificationHandler { let body: [String: Any] = ["notification": ["event_id": rtcNotifyEventID, "room_id": roomID, "prio": "high", - "devices": [["app_id": settings.voipPusherAppID, + "devices": [["app_id": userSettings.app.voipPusherAppID, "pushkey": token, "data": ["format": "event_id_only", "default_payload": defaultPayload]]]]]