Use native video call picture in picture! (#3190)

* Use native video call picture in picture! * Handle isSupported, isPossible and web view errors. * Use new canEnterPip method. * Hide a room's Call button when already joined to the call. * Tweak feature flag description. * PR comments.
2026-11-05 11:58:32 +00:00 · 2024-08-21 17:48:57 +01:00
parent 4e3ab9c04a
commit 304070e8cc
23 changed files with 256 additions and 66 deletions
@@ -27,7 +27,7 @@ struct CallScreenCoordinatorParameters {

 enum CallScreenCoordinatorAction {
    /// The call is still ongoing but the user wishes to navigate around the app.
-    case pictureInPictureStarted(AVPictureInPictureController?)
+    case pictureInPictureStarted(AVPictureInPictureController)
    /// The call is hidden and the user wishes to return to it.
    case pictureInPictureStopped
    /// The call is finished and the screen is done with.
@@ -18,7 +18,7 @@ import AVKit
 import Foundation

 enum CallScreenViewModelAction {
-    case pictureInPictureStarted(AVPictureInPictureController?)
+    case pictureInPictureStarted(AVPictureInPictureController)
    case pictureInPictureStopped
    case dismiss
 }
@@ -36,6 +36,7 @@ struct CallScreenViewState: BindableState {
 struct Bindings {
    var javaScriptMessageHandler: ((Any) -> Void)?
    var javaScriptEvaluator: ((String) async throws -> Any)?
+    var requestPictureInPictureHandler: (() async -> Result<AVPictureInPictureController, CallScreenError>)?
    
    var alertInfo: AlertInfo<UUID>?
 }
@@ -43,4 +44,10 @@ struct Bindings {
 enum CallScreenViewAction {
    case urlChanged(URL?)
    case navigateBack
+    case pictureInPictureWillStop
+    case endCall
+}
+
+enum CallScreenError: Error {
+    case pictureInPictureNotAvailable
 }
@@ -73,8 +73,8 @@ class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol
                
                switch action {
                case let .setAudioEnabled(enabled, roomID):
-                    guard roomID == configuration.callID else {
-                        MXLog.error("Received mute request for a different room: \(roomID) != \(configuration.callID)")
+                    guard roomID == configuration.callRoomID else {
+                        MXLog.error("Received mute request for a different room: \(roomID) != \(configuration.callRoomID)")
                        return
                    }
                    
@@ -107,28 +107,11 @@ class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol
                case .callEnded:
                    actionsSubject.send(.dismiss)
                case .mediaStateChanged(let audioEnabled, _):
-                    elementCallService.setAudioEnabled(audioEnabled, roomID: configuration.callID)
+                    elementCallService.setAudioEnabled(audioEnabled, roomID: configuration.callRoomID)
                }
            }
            .store(in: &cancellables)
        
-        // Use did start otherwise there's a black box left on the screen during the pip controller animation.
-        NotificationCenter.default.publisher(for: .init("AVPictureInPictureControllerDidStartNotification"))
-            .sink { [weak self] notification in
-                guard let self else { return }
-                let controller = notification.object as? AVPictureInPictureController
-                actionsSubject.send(.pictureInPictureStarted(controller))
-            }
-            .store(in: &cancellables)
-        
-        NotificationCenter.default.publisher(for: .init("AVPictureInPictureControllerWillStopNotification"))
-            .sink { [weak self] _ in
-                guard let self else { return }
-                actionsSubject.send(.pictureInPictureStopped)
-                Task { try await self.state.bindings.javaScriptEvaluator?("controls.disableCompatPip()") }
-            }
-            .store(in: &cancellables)
-        
        setupCall()
    }
    
@@ -138,7 +121,11 @@ class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol
            guard let url else { return }
            MXLog.info("URL changed to: \(url)")
        case .navigateBack:
-            handleBackwardsNavigation()
+            Task { await handleBackwardsNavigation() }
+        case .pictureInPictureWillStop:
+            actionsSubject.send(.pictureInPictureStopped)
+        case .endCall:
+            actionsSubject.send(.dismiss)
        }
    }
    
@@ -200,26 +187,19 @@ class CallScreenViewModel: CallScreenViewModelType, CallScreenViewModelProtocol
        }
    }
    
-    private func handleBackwardsNavigation() {
-        #if targetEnvironment(simulator)
-        if UIDevice.current.isPhone {
-            MXLog.warning("The iPhone simulator doesn't support PiP.")
-            actionsSubject.send(.dismiss)
-            return
-        }
-        #endif
-        
-        guard isPictureInPictureEnabled, state.url != nil else {
+    private func handleBackwardsNavigation() async {
+        guard state.url != nil,
+              isPictureInPictureEnabled,
+              let requestPictureInPictureHandler = state.bindings.requestPictureInPictureHandler else {
            actionsSubject.send(.dismiss)
            return
        }
        
-        Task {
-            try await state.bindings.javaScriptEvaluator?("controls.enableCompatPip()")
-            // Enable this check when implemented on web.
-            // if result as? Bool != true {
-            //    actionsSubject.send(.dismiss)
-            // }
+        switch await requestPictureInPictureHandler() {
+        case .success(let controller):
+            actionsSubject.send(.pictureInPictureStarted(controller))
+        case .failure:
+            actionsSubject.send(.dismiss)
        }
    }
    
@@ -14,6 +14,7 @@
 // limitations under the License.
 //

+import AVKit
 import Combine
 import SFSafeSymbols
 import SwiftUI
@@ -46,7 +47,7 @@ struct CallScreen: View {
        if context.viewState.url == nil {
            ProgressView()
        } else {
-            WebView(url: context.viewState.url, viewModelContext: context)
+            CallView(url: context.viewState.url, viewModelContext: context)
                // This URL is stable, forces view reloads if this representable is ever reused for another url
                .id(context.viewState.url)
                .ignoresSafeArea(edges: .bottom)
@@ -54,44 +55,52 @@ struct CallScreen: View {
    }
 }

-private struct WebView: UIViewRepresentable {
+private struct CallView: UIViewRepresentable {
+    /// The top-level view this representable displays. It wraps the web view when picture in picture isn't running.
+    typealias WebViewWrapper = UIView
+    
    let url: URL?
    let viewModelContext: CallScreenViewModel.Context
    
-    func makeUIView(context: Context) -> WKWebView {
-        context.coordinator.webView
+    func makeUIView(context: Context) -> WebViewWrapper {
+        context.coordinator.webViewWrapper
    }
    
    func makeCoordinator() -> Coordinator {
        Coordinator(viewModelContext: viewModelContext)
    }
    
-    func updateUIView(_ webView: WKWebView, context: Context) {
+    func updateUIView(_ callWebView: WebViewWrapper, context: Context) {
        if let url {
-            context.coordinator.url = url
-            let request = URLRequest(url: url)
-            webView.load(request)
+            context.coordinator.load(url)
        }
    }
    
    @MainActor
-    class Coordinator: NSObject, WKUIDelegate, WKNavigationDelegate {
+    class Coordinator: NSObject, WKUIDelegate, WKNavigationDelegate, AVPictureInPictureControllerDelegate {
        private weak var viewModelContext: CallScreenViewModel.Context?
        private let certificateValidator: CertificateValidatorHookProtocol
        
-        private(set) var webView: WKWebView!
+        private var webView: WKWebView!
+        private var pictureInPictureController: AVPictureInPictureController?
+        private let pictureInPictureViewController: AVPictureInPictureVideoCallViewController
        
-        var url: URL!
+        /// The view to be shown in the app. This will contain the web view when picture in picture isn't running.
+        let webViewWrapper = WebViewWrapper(frame: .zero)
+        
+        private var url: URL!
        
        init(viewModelContext: CallScreenViewModel.Context) {
            self.viewModelContext = viewModelContext
            certificateValidator = viewModelContext.viewState.certificateValidator
+            pictureInPictureViewController = AVPictureInPictureVideoCallViewController()
+            pictureInPictureViewController.preferredContentSize = CGSize(width: 1920, height: 1080)
            
            super.init()
            
-            DispatchQueue.main.async {
-                // Avoid `Publishing changes from within view update warnings`
-                viewModelContext.javaScriptEvaluator = self.evaluateJavaScript(_:)
+            DispatchQueue.main.async { // Avoid `Publishing changes from within view update warnings`
+                viewModelContext.javaScriptEvaluator = self.evaluateJavaScript
+                viewModelContext.requestPictureInPictureHandler = self.requestPictureInPicture
            }
            
            let configuration = WKWebViewConfiguration()
@@ -120,6 +129,21 @@ private struct WebView: UIViewRepresentable {
            webView.isOpaque = false
            webView.backgroundColor = .compound.bgCanvasDefault
            webView.scrollView.backgroundColor = .compound.bgCanvasDefault
+            
+            webViewWrapper.addMatchedSubview(webView)
+            
+            if AVPictureInPictureController.isPictureInPictureSupported() {
+                let pictureInPictureController = AVPictureInPictureController(contentSource: .init(activeVideoCallSourceView: webViewWrapper,
+                                                                                                   contentViewController: pictureInPictureViewController))
+                pictureInPictureController.delegate = self
+                self.pictureInPictureController = pictureInPictureController
+            }
+        }
+        
+        func load(_ url: URL) {
+            self.url = url
+            let request = URLRequest(url: url)
+            webView.load(request)
        }
        
        func evaluateJavaScript(_ script: String) async throws -> Any? {
@@ -181,6 +205,57 @@ private struct WebView: UIViewRepresentable {
                viewModelContext?.send(viewAction: .urlChanged(webView.url))
            }
        }
+        
+        // MARK: - Picture in Picture
+        
+        func requestPictureInPicture() async -> Result<AVPictureInPictureController, CallScreenError> {
+            guard let pictureInPictureController,
+                  pictureInPictureController.isPictureInPicturePossible,
+                  case .success(true) = await webViewCanEnterPictureInPicture() else {
+                return .failure(.pictureInPictureNotAvailable)
+            }
+            
+            pictureInPictureController.startPictureInPicture()
+            return .success(pictureInPictureController)
+        }
+        
+        func stopPictureInPicture() {
+            pictureInPictureController?.stopPictureInPicture()
+        }
+        
+        nonisolated func pictureInPictureControllerWillStartPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) {
+            Task { @MainActor in
+                // We move the view via the delegate so it works when you background the app without calling requestPictureInPicture
+                pictureInPictureViewController.view.addMatchedSubview(webView)
+                _ = try? await evaluateJavaScript("controls.enablePip()")
+            }
+        }
+        
+        nonisolated func pictureInPictureControllerWillStopPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) {
+            Task { await viewModelContext?.send(viewAction: .pictureInPictureWillStop) }
+        }
+        
+        nonisolated func pictureInPictureControllerDidStopPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) {
+            Task { @MainActor in
+                webViewWrapper.addMatchedSubview(webView)
+                _ = try? await evaluateJavaScript("controls.disablePip()")
+            }
+        }
+        
+        /// Whether the web view can do picture in picture or not (e.g. it is showing an error or the page didn't load).
+        private func webViewCanEnterPictureInPicture() async -> Result<Bool, CallScreenError> {
+            do {
+                guard let canEnterPictureInPicture = try await evaluateJavaScript("controls.canEnterPip()") as? Bool else {
+                    MXLog.error("canEnterPip returned an unexpected value, skipping picture in picture.")
+                    return .failure(.pictureInPictureNotAvailable)
+                }
+                MXLog.info("canEnterPip returned \(canEnterPictureInPicture)")
+                return .success(canEnterPictureInPicture)
+            } catch {
+                MXLog.error("Error checking canEnterPip: \(error)")
+                return .failure(.pictureInPictureNotAvailable)
+            }
+        }
    }
    
    /// Avoids retain loops between the configuration and webView coordinator
@@ -28,6 +28,7 @@ struct RoomScreenCoordinatorParameters {
    let voiceMessageMediaManager: VoiceMessageMediaManagerProtocol
    let emojiProvider: EmojiProviderProtocol
    let completionSuggestionService: CompletionSuggestionServiceProtocol
+    let ongoingCallRoomIDPublisher: CurrentValuePublisher<String?, Never>
    let appMediator: AppMediatorProtocol
    let appSettings: AppSettings
    let composerDraftService: ComposerDraftServiceProtocol
@@ -64,6 +65,7 @@ final class RoomScreenCoordinator: CoordinatorProtocol {
    init(parameters: RoomScreenCoordinatorParameters) {
        roomViewModel = RoomScreenViewModel(roomProxy: parameters.roomProxy,
                                            mediaProvider: parameters.mediaProvider,
+                                            ongoingCallRoomIDPublisher: parameters.ongoingCallRoomIDPublisher,
                                            appMediator: parameters.appMediator,
                                            appSettings: ServiceLocator.shared.settings,
                                            analyticsService: ServiceLocator.shared.analytics)
@@ -46,6 +46,7 @@ struct RoomScreenViewState: BindableState {
    
    var canJoinCall = false
    var hasOngoingCall: Bool
+    var shouldShowCallButton = true
    
    var bindings: RoomScreenViewStateBindings
 }
@@ -53,6 +53,7 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
    
    init(roomProxy: JoinedRoomProxyProtocol,
         mediaProvider: MediaProviderProtocol,
+         ongoingCallRoomIDPublisher: CurrentValuePublisher<String?, Never>,
         appMediator: AppMediatorProtocol,
         appSettings: AppSettings,
         analyticsService: AnalyticsService) {
@@ -68,7 +69,7 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
                                           bindings: .init()),
                   mediaProvider: mediaProvider)
        
-        setupSubscriptions()
+        setupSubscriptions(ongoingCallRoomIDPublisher: ongoingCallRoomIDPublisher)
    }
    
    override func process(viewAction: RoomScreenViewAction) {
@@ -93,7 +94,7 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
        state.lastScrollDirection = direction
    }
    
-    private func setupSubscriptions() {
+    private func setupSubscriptions(ongoingCallRoomIDPublisher: CurrentValuePublisher<String?, Never>) {
        let roomInfoSubscription = roomProxy
            .actionsPublisher
            .filter { $0 == .roomInfoUpdate }
@@ -135,6 +136,14 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
                self?.setupPinnedEventsTimelineProviderIfNeeded()
            }
            .store(in: &cancellables)
+        
+        ongoingCallRoomIDPublisher
+            .receive(on: DispatchQueue.main)
+            .sink { [weak self] ongoingCallRoomID in
+                guard let self else { return }
+                state.shouldShowCallButton = ongoingCallRoomID != roomProxy.id
+            }
+            .store(in: &cancellables)
    }
    
    private func buildPinnedEventContents(timelineItems: [TimelineItemProxy]) {
@@ -186,6 +195,7 @@ extension RoomScreenViewModel {
    static func mock(roomProxyMock: JoinedRoomProxyMock) -> RoomScreenViewModel {
        RoomScreenViewModel(roomProxy: roomProxyMock,
                            mediaProvider: MockMediaProvider(),
+                            ongoingCallRoomIDPublisher: .init(.init(nil)),
                            appMediator: AppMediatorMock.default,
                            appSettings: ServiceLocator.shared.settings,
                            analyticsService: ServiceLocator.shared.analytics)
@@ -175,8 +175,10 @@ struct RoomScreen: View {
        
        if !ProcessInfo.processInfo.isiOSAppOnMac {
            ToolbarItem(placement: .primaryAction) {
-                callButton
-                    .disabled(roomContext.viewState.canJoinCall == false)
+                if roomContext.viewState.shouldShowCallButton {
+                    callButton
+                        .disabled(!roomContext.viewState.canJoinCall)
+                }
            }
        }
    }
@@ -53,6 +53,7 @@ protocol DeveloperOptionsProtocol: AnyObject {
    var elementCallBaseURLOverride: URL? { get set }
    var fuzzyRoomListSearchEnabled: Bool { get set }
    var pinningEnabled: Bool { get set }
+    var elementCallPictureInPictureEnabled: Bool { get set }
 }

 extension AppSettings: DeveloperOptionsProtocol { }
@@ -61,6 +61,11 @@ struct DeveloperOptionsScreen: View {
                    .autocorrectionDisabled(true)
                    .autocapitalization(.none)
                    .foregroundColor(URL(string: elementCallBaseURLString) == nil ? .red : .primary)
+                
+                Toggle(isOn: $context.elementCallPictureInPictureEnabled) {
+                    Text("Picture in Picture support")
+                    Text("Requires an Element Call deployment with support for signalling PiP availability.")
+                }
            }
            
            Section {