Add experimental native calls behind a developer flag (#6159)

Add the element-call-ios dependency and a developer option for native calls The package carries the call UI and the matrix-rust-rtc media stack, pinned by exact version the way the SDK is, with the local path kept commented out for working against a checkout. Its media xcframework has no Intel simulator slice and a package cannot say so for the projects that consume it, hence the excluded architecture at the project level. The developer option is off by default and nothing reads it yet. It is placed next to the Element Call URL override, which only applies to the web view that native calls replace.
2026-11-03 06:55:59 +00:00 · 2026-09-17 18:42:18 +02:00
parent fe18472556
commit fb49f5b83b
33 changed files with 1778 additions and 160 deletions
@@ -9,6 +9,7 @@
 import AVFoundation
 import CallKit
 import Combine
+import ElementCall
 import Foundation
 import MatrixRustSDK
 import PushKit
@@ -26,10 +27,12 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
        let roomID: String
        let rtcNotificationID: String?
        let isVoiceCall: Bool
+        var isNative = false
    }
    
+    private let appSettings: AppSettings
    private let pushRegistry: PKPushRegistry
-    private let callController = CXCallController()
+    private let callController: CXCallControllerProtocol
    private let callProvider: CXProviderProtocol
    private let timeProvider: TimeProvider
    
@@ -51,6 +54,10 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
    
    private var endUnansweredCallTask: Task<Void, Never>?
    
+    /// A call answered for the native stack, until the stack takes it over.
+    private var answeredNativeCallID: CallID?
+    private var endPendingCallTask: Task<Void, Never>?
+    
    private var ongoingCallID: CallID? {
        didSet { ongoingCallRoomIDSubject.send(ongoingCallID?.roomID) }
    }
@@ -67,7 +74,23 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
    
    private var declineListenerHandle: TaskHandle?
    
-    init(callProvider: CXProviderProtocol? = nil, timeProvider: TimeProvider? = nil) {
+    /// The native call stack, for as long as there's a session running calls through it.
+    private var nativeCallStack: NativeCallStack?
+    private var nativeCallCancellable: AnyCancellable?
+    private var mediaProvider: MediaProviderProtocol?
+    
+    var nativeCallController: ElementCallController? {
+        nativeCallStack?.controller
+    }
+    
+    private var cancellables = Set<AnyCancellable>()
+    
+    init(appSettings: AppSettings,
+         callProvider: CXProviderProtocol? = nil,
+         callController: CXCallControllerProtocol? = nil,
+         timeProvider: TimeProvider? = nil) {
+        self.appSettings = appSettings
+        self.callController = callController ?? CXCallController()
        pushRegistry = PKPushRegistry(queue: nil)
        
        self.timeProvider = timeProvider ?? TimeProvider(clock: ContinuousClock(), now: Date.init)
@@ -95,45 +118,130 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
        pushRegistry.desiredPushTypes = [.voIP]
        
        self.callProvider.setDelegate(self, queue: nil)
+        
+        appSettings.nativeCallEnabledPublisher
+            .sink { [weak self] _ in
+                self?.updateNativeCallStack()
+            }
+            .store(in: &cancellables)
    }
    
-    func setClientProxy(_ clientProxy: any ClientProxyProtocol) {
-        self.clientProxy = clientProxy
+    func setUserSession(_ userSession: UserSessionProtocol?) {
+        // A stack is built against one client proxy and can't be handed to another, which is what a
+        // soft logout or a cache clear brings: a new session without the sign out that drops this one.
+        if userSession?.clientProxy !== clientProxy {
+            stopNativeCallStack()
+        }
+        
+        clientProxy = userSession?.clientProxy
+        mediaProvider = userSession?.mediaProvider
+        
+        updateNativeCallStack()
    }
    
-    func setupCallSession(roomID: String, roomDisplayName: String) async {
+    @discardableResult
+    func handleNativeCallRequest(roomProxy: JoinedRoomProxyProtocol, isVoiceCall: Bool) -> Bool {
+        guard let nativeCallStack else {
+            return false
+        }
+        
+        nativeCallStack.handleCallRequest(roomProxy: roomProxy, isVoiceCall: isVoiceCall)
+        return true
+    }
+    
+    func minimizeNativeCall() {
+        nativeCallStack?.minimize()
+    }
+    
+    func restoreNativeCall() {
+        nativeCallStack?.restore()
+    }
+    
+    func setupCallSession(roomID: String, roomDisplayName: String, isVideo: Bool) async {
+        // Unlike the push handling below, this always runs with a session, so the stack itself is
+        // the answer rather than the setting that would have built one.
+        let isNative = nativeCallStack != nil
+        MXLog.info("Setting up a call session, native: \(isNative), video: \(isVideo), replacing an ongoing call: \(ongoingCallID != nil)")
+        
        // Drop any ongoing calls when starting a new one
        if ongoingCallID != nil {
-            tearDownCallSession()
+            tearDownCallSession(sendEndCallAction: true)
        }
        
        // If this starting from a ring reuse those identifiers
        // Make sure the roomID matches
-        let callID = if let incomingCallID, incomingCallID.roomID == roomID {
+        var callID = if let incomingCallID, incomingCallID.roomID == roomID {
            incomingCallID
        } else {
-            CallID(callKitID: UUID(), roomID: roomID, rtcNotificationID: nil, isVoiceCall: false)
+            CallID(callKitID: UUID(), roomID: roomID, rtcNotificationID: nil, isVoiceCall: !isVideo)
        }
+        callID.isNative = isNative
+        let isAnsweringIncomingCall = callID.rtcNotificationID != nil
        
        clearIncomingCallState()
        ongoingCallID = callID
        
-        // Don't bother starting another CallKit session as it won't work properly
+        // A web view call must not be tracked by CallKit, as that gives this process exclusive
+        // access to media and the web view runs in another (see `CXAnswerCallAction`).
        // https://developer.apple.com/forums//thread/767949?answerId=812951022#812951022
+        guard isNative else {
+            return
+        }
        
-        // let handle = CXHandle(type: .generic, value: roomDisplayName)
-        // let startCallAction = CXStartCallAction(call: callID.callKitID, handle: handle)
-        // startCallAction.isVideo = true
+        // Answering the ringing call: it is already reported, keep it.
+        guard !isAnsweringIncomingCall else {
+            return
+        }
        
-        // do {
-        //     try await callController.request(CXTransaction(action: startCallAction))
-        // } catch {
-        //     MXLog.error("Failed requesting start call action with error: \(error)")
-        // }
+        let handle = CXHandle(type: .generic, value: roomID)
+        let startCallAction = CXStartCallAction(call: callID.callKitID, handle: handle)
+        startCallAction.contactIdentifier = roomDisplayName
+        startCallAction.isVideo = isVideo
+        
+        // Awaited because our membership goes out straight after: a peer that sees it before the
+        // system knows about the call reads the call as answered elsewhere.
+        await requestTransaction(CXTransaction(action: startCallAction), describedAs: "start call")
+        
+        // Without this the system call UI shows the handle, i.e. the room ID.
+        let update = CXCallUpdate()
+        update.localizedCallerName = roomDisplayName
+        update.remoteHandle = handle
+        update.hasVideo = isVideo
+        callProvider.reportCall(with: callID.callKitID, updated: update)
    }
    
-    func tearDownCallSession() {
-        tearDownCallSession(sendEndCallAction: true)
+    func reportCallSessionConnected(roomID: String) {
+        guard let ongoingCallID, ongoingCallID.roomID == roomID else {
+            // The system ends an outgoing call that never reports connecting, so a silent guard
+            // here shows up much later as a call that hangs itself up.
+            MXLog.warning("Not reporting the call as connected, no matching ongoing call")
+            return
+        }
+        
+        guard ongoingCallID.isNative else {
+            MXLog.info("Not reporting a web view call as connected, CallKit doesn't track it")
+            return
+        }
+        
+        MXLog.info("Reporting the call as connected")
+        callProvider.reportOutgoingCall(with: ongoingCallID.callKitID, connectedAt: nil)
+    }
+    
+    func tearDownCallSession(roomID: String) {
+        if ongoingCallID?.roomID == roomID {
+            tearDownCallSession(sendEndCallAction: true)
+            return
+        }
+        
+        // Answered and handed over, but the stack couldn't take it. Ending it here rather than
+        // waiting for the watchdog takes the system call UI away straight away.
+        if answeredNativeCallID?.roomID == roomID {
+            MXLog.warning("Ending an answered call the native call stack didn't take over")
+            endPendingCall()
+            return
+        }
+        
+        MXLog.info("Not tearing down the call session, no call for room \(roomID)")
    }
    
    func setAudioEnabled(_ enabled: Bool, roomID: String) {
@@ -155,6 +263,57 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
        }
    }
    
+    // MARK: - Native calls
+    
+    /// Builds or releases the call stack to match the session and the setting.
+    private func updateNativeCallStack() {
+        guard appSettings.nativeCallEnabled, let clientProxy else {
+            stopNativeCallStack()
+            return
+        }
+        
+        guard nativeCallStack == nil else { return }
+        
+        guard let transport = clientProxy.makeNativeCallTransport() else {
+            MXLog.error("Cannot start the native call stack without a transport")
+            return
+        }
+        
+        let style = ElementCallStyle(theme: NativeCallTheme(),
+                                     icons: NativeCallIcons(),
+                                     avatars: NativeCallAvatars(mediaProvider: mediaProvider),
+                                     strings: .init(you: L10n.commonYou,
+                                                    error: L10n.commonError,
+                                                    stop: L10n.actionStop,
+                                                    back: L10n.actionBack))
+        
+        // The stats overlay is raw RTP counters, so it follows whatever already reveals developer
+        // surface. Screen sharing stays off until the broadcast extension that makes it work lands.
+        let options = ElementCallOptions(isDeveloperModeEnabled: appSettings.developerOptionsEnabled)
+        
+        let stack = NativeCallStack(transport: transport,
+                                    system: NativeCallSystem(service: self),
+                                    options: options,
+                                    style: style)
+        nativeCallStack = stack
+        
+        nativeCallCancellable = stack.actions
+            .sink { [weak self] presentation in
+                self?.actionsSubject.send(.nativeCall(presentation))
+            }
+        
+        Task { await stack.start() }
+    }
+    
+    private func stopNativeCallStack() {
+        guard let nativeCallStack else { return }
+        self.nativeCallStack = nil
+        nativeCallCancellable = nil
+        
+        nativeCallStack.stop()
+        actionsSubject.send(.nativeCall(.dismiss))
+    }
+    
    // MARK: - PKPushRegistryDelegate
    
    func pushRegistry(_ registry: PKPushRegistry, didUpdate pushCredentials: PKPushCredentials, for type: PKPushType) { }
@@ -215,7 +374,7 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
        // If not for audio call the app will not be put to foreground and the webview won't be able to handle the call...
        // Consequence: The call will be presented to the user as a video call in CallKit UI,
        // but once Element Call is launched it will correctly route to a voice-only call.
-        update.hasVideo = true
+        update.hasVideo = appSettings.nativeCallEnabled ? !isVoiceCall : true
        update.localizedCallerName = roomDisplayName
        // https://stackoverflow.com/a/41230020/730924
        update.remoteHandle = .init(type: .generic, value: roomID)
@@ -253,10 +412,19 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
    
    func provider(_ provider: CXProvider, didActivate audioSession: AVAudioSession) {
        MXLog.info("Call provider did activate audio session")
+        actionsSubject.send(.audioSessionActivated)
    }
    
    func provider(_ provider: CXProvider, didDeactivate audioSession: AVAudioSession) {
        MXLog.info("Call provider did deactivate audio session")
+        actionsSubject.send(.audioSessionDeactivated)
+    }
+    
+    func provider(_ provider: CXProvider, perform action: CXStartCallAction) {
+        // The native controller configured the audio session already; CallKit activates it and
+        // reports back through `didActivate`.
+        action.fulfill()
+        callProvider.reportOutgoingCall(with: action.callUUID, startedConnectingAt: nil)
    }
    
    func providerDidReset(_ provider: CXProvider) {
@@ -287,6 +455,15 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
        // First fullfill the action
        action.fulfill()
        
+        if appSettings.nativeCallEnabled {
+            // The native stack owns media in this process, so the CallKit call stays up: the
+            // controller attaches to it in `setupCallSession`.
+            endUnansweredCallTask?.cancel()
+            endCallIfLeftPending(incomingCallID)
+            actionsSubject.send(.startCall(roomID: incomingCallID.roomID, isVoiceCall: incomingCallID.isVoiceCall))
+            return
+        }
+        
        // And delay ending the call so that the app has enough time
        // to get deeplinked into
        DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) {
@@ -302,7 +479,7 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
        if let ongoingCallID {
            actionsSubject.send(.setAudioEnabled(!action.isMuted, roomID: ongoingCallID.roomID))
        } else {
-            MXLog.error("Failed muting/unmuting call, missing ongoingCallID")
+            MXLog.info("Ignoring a mute action without an ongoing call")
        }
        
        action.fulfill()
@@ -313,30 +490,56 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
        // This gets called for no reason on simulators, where CallKit
        // isn't even supported, ignore it.
        #else
-        if let ongoingCallID {
-            actionsSubject.send(.endCall(roomID: ongoingCallID.roomID))
-        }
-        
-        if let incomingCallID {
-            Task {
-                await sendDeclineCallEvent(incomingCallID)
-            }
-        }
-        
-        if incomingCallID?.callKitID == action.callUUID {
-            clearIncomingCallState()
-        }
-        
-        tearDownCallSession(sendEndCallAction: false)
-        
+        endCall(withCallKitID: action.callUUID)
        action.fulfill()
        #endif
    }
    
+    /// Ends the call the system identified, whether that's the ongoing one, the ringing one, or
+    /// neither because it has already been replaced.
+    ///
+    /// Kept out of the delegate method so that it's reachable on the simulator, where the provider
+    /// performs end call actions of its own and the delegate has to ignore them.
+    func endCall(withCallKitID callKitID: UUID) {
+        MXLog.info("Ending the call for \(callKitID), ongoing: \(ongoingCallID != nil), incoming: \(incomingCallID != nil)")
+        
+        // Both branches are gated on the identifier. Leaving a call to start one in another room
+        // requests an end call action for the old call and replaces `ongoingCallID` before the
+        // system gets round to performing it, so an ungated handler reads it as the new call ending
+        // and hangs up the call that was just started.
+        if let ongoingCallID, ongoingCallID.callKitID == callKitID {
+            actionsSubject.send(.endCall(roomID: ongoingCallID.roomID))
+            tearDownCallSession(sendEndCallAction: false)
+        }
+        
+        if let incomingCallID, incomingCallID.callKitID == callKitID {
+            Task {
+                await sendDeclineCallEvent(incomingCallID)
+            }
+            
+            clearIncomingCallState()
+        }
+    }
+    
    // MARK: - Private
    
+    /// Bridges the controller's completion handler, which is all ``CXCallControllerProtocol`` can
+    /// carry, back to an awaitable call.
+    private func requestTransaction(_ transaction: CXTransaction, describedAs description: String) async {
+        await withCheckedContinuation { continuation in
+            callController.request(transaction) { error in
+                if let error {
+                    MXLog.error("Failed requesting \(description) action with error: \(error)")
+                }
+                
+                continuation.resume()
+            }
+        }
+    }
+    
    private func tearDownCallSession(sendEndCallAction: Bool = true) {
        if sendEndCallAction, let ongoingCallID {
+            MXLog.info("Requesting an end call action for the ongoing call")
            let transaction = CXTransaction(action: CXEndCallAction(call: ongoingCallID.callKitID))
            callController.request(transaction) { error in
                if let error {
@@ -381,6 +584,32 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
        }
    }
    
+    /// Ends an answered call that the native stack never took over, which nothing else would: the
+    /// call is deliberately left up for the controller to attach to in `setupCallSession`.
+    private func endCallIfLeftPending(_ callID: CallID) {
+        answeredNativeCallID = callID
+        
+        endPendingCallTask = Task { [weak self] in
+            try? await self?.timeProvider.clock.sleep(for: .seconds(30))
+            
+            guard let self, !Task.isCancelled, answeredNativeCallID == callID else {
+                return
+            }
+            
+            MXLog.error("The native call stack never took over the answered call, ending it")
+            endPendingCall()
+        }
+    }
+    
+    private func endPendingCall() {
+        guard let answeredNativeCallID else {
+            return
+        }
+        
+        callProvider.reportCall(with: answeredNativeCallID.callKitID, endedAt: nil, reason: .failed)
+        clearIncomingCallState()
+    }
+    
    private func sendDeclineCallEvent(_ incomingCallID: CallID) async {
        guard let rtcNotificationID = incomingCallID.rtcNotificationID else {
            MXLog.info("No rtc notification event to decline.")
@@ -482,6 +711,9 @@ class ElementCallService: NSObject, ElementCallServiceProtocol, PKPushRegistryDe
    private func clearIncomingCallState() {
        endUnansweredCallTask?.cancel()
        endUnansweredCallTask = nil
+        endPendingCallTask?.cancel()
+        endPendingCallTask = nil
+        answeredNativeCallID = nil
        declineListenerHandle?.cancel()
        declineListenerHandle = nil
        incomingCallRoomInfoCancellable = nil