Performance tests (#1301)

* Add signposts to performance tests.

- Update flow to include support for the migration screen.

* If the welcome screen shows, click on the button.

* Ensure a clean simulator each run.

* Add accessibility identifier for migration screen if required.

* Handle walking into the room and back out again.

* use iphone 14 pro to match what's used in xcode.

* Remove ApplicationTests as duplicated in LoginTests.

We measure app startup time in LoginTests as part of the flow - we may as well avoid spending 60s doing only that measurement in ApplicationTests

* Sleep 10s, the ui is otherwise showing up in random order.

* Revert "Remove ApplicationTests as duplicated in LoginTests."

This reverts commit 8670710315bcd0d6c3c3046f534b32b4c728b837.

* Update script to parse out correct values from results file.

* Allow cancellation of password prompt in any order.

* Remove test timeout, performance tests will always take a while.

* Adjust parsing further

* Remove ApplicationTests.

* Move to a more elegant way to wait for something to disappear.

* Linting.

* Fix unit tests.

---------

Co-authored-by: Doug <douglase@element.io>
Co-authored-by: Doug <6060466+pixlwave@users.noreply.github.com>

ElementX/Sources/Application/AppCoordinator.swift

@@ -353,7 +353,8 @@ class AppCoordinator: AppCoordinatorProtocol, AuthenticationCoordinatorDelegate,
                                                                     navigationSplitCoordinator: navigationSplitCoordinator,
                                                                     bugReportService: ServiceLocator.shared.bugReportService,
                                                                     roomTimelineControllerFactory: RoomTimelineControllerFactory(),
-                                                                    appSettings: ServiceLocator.shared.settings)
+                                                                    appSettings: ServiceLocator.shared.settings,
+                                                                    analytics: ServiceLocator.shared.analytics)
         
         userSessionFlowCoordinator.callback = { [weak self] action in
             switch action {
@@ -550,6 +551,7 @@ class AppCoordinator: AppCoordinatorProtocol, AuthenticationCoordinatorDelegate,
     }
 
     private func startSync() {
+        ServiceLocator.shared.analytics.signpost.beginSync()
         guard let userSession else {
             fatalError("User session not setup")
         }
@@ -574,6 +576,7 @@ class AppCoordinator: AppCoordinatorProtocol, AuthenticationCoordinatorDelegate,
                 case .startedUpdating:
                     showLoadingIndicator()
                 case .receivedSyncUpdate:
+                    ServiceLocator.shared.analytics.signpost.endSync()
                     ServiceLocator.shared.userIndicatorController.retractIndicatorWithId(identifier)
                 default:
                     break

ElementX/Sources/FlowCoordinators/UserSessionFlowCoordinator.swift

@@ -28,6 +28,7 @@ class UserSessionFlowCoordinator: FlowCoordinatorProtocol {
     private let bugReportService: BugReportServiceProtocol
     private let roomTimelineControllerFactory: RoomTimelineControllerFactoryProtocol
     private let appSettings: AppSettings
+    private let analytics: AnalyticsService
     
     private let stateMachine: UserSessionFlowCoordinatorStateMachine
     private let roomFlowCoordinator: RoomFlowCoordinator
@@ -46,13 +47,15 @@ class UserSessionFlowCoordinator: FlowCoordinatorProtocol {
          navigationSplitCoordinator: NavigationSplitCoordinator,
          bugReportService: BugReportServiceProtocol,
          roomTimelineControllerFactory: RoomTimelineControllerFactoryProtocol,
-         appSettings: AppSettings) {
+         appSettings: AppSettings,
+         analytics: AnalyticsService) {
         stateMachine = UserSessionFlowCoordinatorStateMachine()
         self.userSession = userSession
         self.navigationSplitCoordinator = navigationSplitCoordinator
         self.bugReportService = bugReportService
         self.roomTimelineControllerFactory = roomTimelineControllerFactory
         self.appSettings = appSettings
+        self.analytics = analytics
         
         sidebarNavigationStackCoordinator = NavigationStackCoordinator(navigationSplitCoordinator: navigationSplitCoordinator)
         detailNavigationStackCoordinator = NavigationStackCoordinator(navigationSplitCoordinator: navigationSplitCoordinator)
@@ -64,8 +67,8 @@ class UserSessionFlowCoordinator: FlowCoordinatorProtocol {
                                                   navigationStackCoordinator: detailNavigationStackCoordinator,
                                                   navigationSplitCoordinator: navigationSplitCoordinator,
                                                   emojiProvider: EmojiProvider(),
-                                                  appSettings: ServiceLocator.shared.settings,
-                                                  analytics: ServiceLocator.shared.analytics,
+                                                  appSettings: appSettings,
+                                                  analytics: analytics,
                                                   userIndicatorController: ServiceLocator.shared.userIndicatorController)
         
         setupStateMachine()
@@ -73,9 +76,11 @@ class UserSessionFlowCoordinator: FlowCoordinatorProtocol {
         roomFlowCoordinator.actions.sink { action in
             switch action {
             case .presentedRoom(let roomID):
+                self.analytics.signpost.beginRoomFlow(roomID)
                 self.stateMachine.processEvent(.selectRoom(roomId: roomID))
             case .dismissedRoom:
                 self.stateMachine.processEvent(.deselectRoom)
+                self.analytics.signpost.endRoomFlow()
             }
         }
         .store(in: &cancellables)

ElementX/Sources/Other/AccessibilityIdentifiers.swift

@@ -33,7 +33,9 @@ struct A11yIdentifiers {
     static let roomMemberDetailsScreen = RoomMemberDetailsScreen()
     static let createRoomScreen = CreateRoomScreen()
     static let invitesScreen = InvitesScreen()
-    
+    static let welcomeScreen = WelcomeScreen()
+    static let migrationScreen = MigrationScreen()
+
     struct AnalyticsPromptScreen {
         let title = "analytics_prompt-title"
         let enable = "analytics_prompt-enable"
@@ -59,8 +61,10 @@ struct A11yIdentifiers {
         let settings = "home_screen-settings"
         let signOut = "home_screen-sign_out"
         let verificationBannerContinue = "home_screen-verification_continue"
+        
+        let roomNamePrefix = "home_screen-room_name"
         func roomName(_ name: String) -> String {
-            "home_screen-room_name:\(name)"
+            "\(roomNamePrefix):\(name)"
         }
     }
     
@@ -142,4 +146,12 @@ struct A11yIdentifiers {
         let roomName = "create_room-room_name"
         let roomTopic = "create_room-room_topic"
     }
+
+    struct WelcomeScreen {
+        let letsGo = "welcome_screen-lets_go"
+    }
+
+    struct MigrationScreen {
+        let message = "migration_screen-message"
+    }
 }

ElementX/Sources/Other/SwiftUI/Animation/ShimmerModifier.swift

@@ -76,6 +76,7 @@ struct ShimmerOverlay_Previews: PreviewProvider {
                                                attributedStringBuilder: AttributedStringBuilder(permalinkBaseURL: ServiceLocator.shared.settings.permalinkBaseURL),
                                                selectedRoomPublisher: CurrentValueSubject<String?, Never>(nil).asCurrentValuePublisher(),
                                                appSettings: ServiceLocator.shared.settings,
+                                               analytics: ServiceLocator.shared.analytics,
                                                userIndicatorController: ServiceLocator.shared.userIndicatorController)
     
     static var previews: some View {

ElementX/Sources/Screens/Authentication/AuthenticationCoordinator.swift

@@ -165,6 +165,7 @@ class AuthenticationCoordinator: CoordinatorProtocol {
     
     private func showLoginScreen() {
         let parameters = LoginScreenCoordinatorParameters(authenticationService: authenticationService,
+                                                          analytics: analytics,
                                                           userIndicatorController: userIndicatorController)
         let coordinator = LoginScreenCoordinator(parameters: parameters)
         

ElementX/Sources/Screens/Authentication/LoginScreen/LoginScreenCoordinator.swift

@@ -21,6 +21,7 @@ struct LoginScreenCoordinatorParameters {
     /// The service used to authenticate the user.
     let authenticationService: AuthenticationServiceProxyProtocol
     
+    let analytics: AnalyticsService
     let userIndicatorController: UserIndicatorControllerProtocol
 }
 
@@ -119,15 +120,18 @@ final class LoginScreenCoordinator: CoordinatorProtocol {
         startLoading(isInteractionBlocking: true)
         
         Task {
+            parameters.analytics.signpost.beginLogin()
             switch await authenticationService.login(username: username,
                                                      password: password,
                                                      initialDeviceName: UIDevice.current.initialDeviceName,
                                                      deviceID: nil) {
             case .success(let userSession):
                 callback?(.signedIn(userSession))
+                parameters.analytics.signpost.endLogin()
                 stopLoading()
             case .failure(let error):
                 stopLoading()
+                parameters.analytics.signpost.endLogin()
                 switch error {
                 case .isOnWaitlist:
                     callback?(.isOnWaitlist(.init(username: username,

ElementX/Sources/Screens/HomeScreen/HomeScreenCoordinator.swift

@@ -52,6 +52,7 @@ final class HomeScreenCoordinator: CoordinatorProtocol {
                                         attributedStringBuilder: parameters.attributedStringBuilder,
                                         selectedRoomPublisher: parameters.selectedRoomPublisher,
                                         appSettings: ServiceLocator.shared.settings,
+                                        analytics: ServiceLocator.shared.analytics,
                                         userIndicatorController: ServiceLocator.shared.userIndicatorController)
         
         viewModel.callback = { [weak self] action in

ElementX/Sources/Screens/HomeScreen/HomeScreenViewModel.swift

@@ -23,6 +23,7 @@ class HomeScreenViewModel: HomeScreenViewModelType, HomeScreenViewModelProtocol
     private let userSession: UserSessionProtocol
     private let attributedStringBuilder: AttributedStringBuilderProtocol
     private let appSettings: AppSettings
+    private let analytics: AnalyticsService
     private let userIndicatorController: UserIndicatorControllerProtocol
     
     private let roomSummaryProvider: RoomSummaryProviderProtocol?
@@ -38,10 +39,12 @@ class HomeScreenViewModel: HomeScreenViewModelType, HomeScreenViewModelProtocol
          attributedStringBuilder: AttributedStringBuilderProtocol,
          selectedRoomPublisher: CurrentValuePublisher<String?, Never>,
          appSettings: AppSettings,
+         analytics: AnalyticsService,
          userIndicatorController: UserIndicatorControllerProtocol) {
         self.userSession = userSession
         self.attributedStringBuilder = attributedStringBuilder
         self.appSettings = appSettings
+        self.analytics = analytics
         self.userIndicatorController = userIndicatorController
         
         roomSummaryProvider = userSession.clientProxy.roomSummaryProvider
@@ -98,6 +101,11 @@ class HomeScreenViewModel: HomeScreenViewModelType, HomeScreenViewModelProtocol
                     return
                 }
                 
+                // End the initial sync performance timing once the room list is about to be displayed.
+                if roomListMode == .rooms, self.state.roomListMode == .skeletons {
+                    analytics.signpost.endSync()
+                }
+                
                 self.state.roomListMode = roomListMode
                 
                 MXLog.info("Received room summary provider update, setting view room list mode to \"\(self.state.roomListMode)\"")

ElementX/Sources/Screens/HomeScreen/View/HomeScreen.swift

@@ -228,6 +228,7 @@ struct HomeScreen_Previews: PreviewProvider {
                                             attributedStringBuilder: AttributedStringBuilder(permalinkBaseURL: ServiceLocator.shared.settings.permalinkBaseURL),
                                             selectedRoomPublisher: CurrentValueSubject<String?, Never>(nil).asCurrentValuePublisher(),
                                             appSettings: ServiceLocator.shared.settings,
+                                            analytics: ServiceLocator.shared.analytics,
                                             userIndicatorController: ServiceLocator.shared.userIndicatorController)
         
         return NavigationStack {

ElementX/Sources/Screens/HomeScreen/View/HomeScreenRoomCell.swift

@@ -177,6 +177,7 @@ struct HomeScreenRoomCell_Previews: PreviewProvider {
                                             attributedStringBuilder: AttributedStringBuilder(permalinkBaseURL: ServiceLocator.shared.settings.permalinkBaseURL),
                                             selectedRoomPublisher: CurrentValueSubject<String?, Never>(nil).asCurrentValuePublisher(),
                                             appSettings: ServiceLocator.shared.settings,
+                                            analytics: ServiceLocator.shared.analytics,
                                             userIndicatorController: ServiceLocator.shared.userIndicatorController)
         
         let rooms: [HomeScreenRoom] = summaryProvider.roomListPublisher.value.compactMap { summary -> HomeScreenRoom? in

ElementX/Sources/Screens/MigrationScreen/View/MigrationScreen.swift

@@ -43,6 +43,7 @@ struct MigrationScreen: View {
                 .font(.compound.bodyLG)
                 .multilineTextAlignment(.center)
                 .foregroundColor(.compound.textPrimary)
+                .accessibilityIdentifier(A11yIdentifiers.migrationScreen.message)
         }
     }
 }

ElementX/Sources/Screens/WelcomeScreenScreen/View/WelcomeScreen.swift

@@ -94,6 +94,7 @@ struct WelcomeScreen: View {
             Text(context.viewState.buttonTitle)
         }
         .buttonStyle(.elementAction(.xLarge))
+        .accessibilityIdentifier(A11yIdentifiers.welcomeScreen.letsGo)
     }
 }
 

ElementX/Sources/Services/Analytics/AnalyticsService.swift

@@ -35,6 +35,10 @@ class AnalyticsService {
     private let client: AnalyticsClientProtocol
     private let appSettings: AppSettings
     private let bugReportService: BugReportServiceProtocol
+    
+    /// A signpost client for performance testing the app. This client doesn't respect the
+    /// `isRunning` state or behave any differently when `start`/`reset` are called.
+    let signpost = Signposter()
 
     init(client: AnalyticsClientProtocol, appSettings: AppSettings, bugReportService: BugReportServiceProtocol) {
         self.client = client

ElementX/Sources/Services/Analytics/Signposter.swift

@@ -0,0 +1,84 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import OSLog
+
+/// A simple wrapper around OSSignposter for easy performance testing.
+class Signposter {
+    /// The underlying signposter.
+    private let signposter = OSSignposter(subsystem: subsystem, category: category)
+    /// A logger instance to capture any errors.
+    private let logger = Logger(subsystem: subsystem, category: category)
+    
+    /// Signpost name constants.
+    enum Name {
+        static let login: StaticString = "Login"
+        static let sync: StaticString = "Sync"
+        static let roomFlow: StaticString = "RoomFlow"
+    }
+    
+    static let subsystem = "ElementX"
+    static let category = "PerformanceTests"
+    
+    // MARK: - Login
+    
+    private var loginState: OSSignpostIntervalState?
+    
+    func beginLogin() {
+        loginState = signposter.beginInterval(Name.login)
+    }
+    
+    func endLogin() {
+        guard let loginState else {
+            logger.error("Missing login state.")
+            return
+        }
+        
+        signposter.endInterval(Name.login, loginState)
+        self.loginState = nil
+    }
+    
+    // MARK: - Sync
+    
+    private var syncState: OSSignpostIntervalState?
+    
+    func beginSync() {
+        syncState = signposter.beginInterval(Name.sync)
+    }
+    
+    func endSync() {
+        guard let syncState else { return }
+        
+        signposter.endInterval(Name.sync, syncState)
+        self.syncState = nil
+    }
+    
+    // MARK: - Room Flow
+    
+    private var roomFlowState: OSSignpostIntervalState?
+    
+    func beginRoomFlow(_ name: String) {
+        roomFlowState = signposter.beginInterval(Name.roomFlow)
+        signposter.emitEvent("RoomName", "\(name, privacy: .auto(mask: .hash))")
+    }
+    
+    func endRoomFlow() {
+        guard let roomFlowState else { return }
+        
+        signposter.endInterval(Name.roomFlow, roomFlowState)
+        self.roomFlowState = nil
+    }
+}

ElementX/Sources/UITests/UITestsAppCoordinator.swift

@@ -78,6 +78,7 @@ class MockScreen: Identifiable {
         case .login:
             let navigationStackCoordinator = NavigationStackCoordinator()
             let coordinator = LoginScreenCoordinator(parameters: .init(authenticationService: MockAuthenticationServiceProxy(),
+                                                                       analytics: ServiceLocator.shared.analytics,
                                                                        userIndicatorController: ServiceLocator.shared.userIndicatorController))
             navigationStackCoordinator.setRootCoordinator(coordinator)
             return navigationStackCoordinator
@@ -336,7 +337,8 @@ class MockScreen: Identifiable {
                                                          navigationSplitCoordinator: navigationSplitCoordinator,
                                                          bugReportService: BugReportServiceMock(),
                                                          roomTimelineControllerFactory: MockRoomTimelineControllerFactory(),
-                                                         appSettings: ServiceLocator.shared.settings)
+                                                         appSettings: ServiceLocator.shared.settings,
+                                                         analytics: ServiceLocator.shared.analytics)
             
             coordinator.start()