Fix the UI tests. (#4085)

* Compile errors 😬

* Update server selection snapshots.

The default footer string has changed.

* Remove the TemplateScreen UI tests.

We use preview tests for snapshots now, UI tests are for flow coordinators.

* Add the same looping on the OIDC UI test (but disable the test in the end).

* Run UI tests separately for iPhone and iPad.

GitHub runners don't seem to have enough memory to handle 2 simulators at once.

.github/workflows/ui_tests.yml

@@ -14,10 +14,13 @@ jobs:
   tests:
     name: Tests
     runs-on: macos-15
+    strategy:
+      matrix:
+        device: [iPhone, iPad]
 
     concurrency:
       # Only allow a single run of this workflow on each branch, automatically cancelling older runs.
-      group: ${{ format('ui-tests-{0}', github.ref) }}
+      group: ${{ format('ui-tests-{0}-{1}', github.ref, matrix.device) }}
       cancel-in-progress: true
 
     steps:
@@ -40,9 +43,9 @@ jobs:
       - name: Run tests
         run: |
           if [[ -z "${{ github.event.inputs.test_name }}" ]]; then
-            bundle exec fastlane ui_tests
+            bundle exec fastlane ui_tests device:${{ matrix.device }} 
           else
-            bundle exec fastlane ui_tests test_name:${{ github.event.inputs.test_name }}
+            bundle exec fastlane ui_tests device:${{ matrix.device }} test_name:${{ github.event.inputs.test_name }}
           fi
 
       - name: Zip results # for faster upload
@@ -54,7 +57,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Results
+          name: ${{ matrix.device }}
           path: fastlane/test_output/UITests.xcresult.zip
           retention-days: 7
           if-no-files-found: ignore

ElementX.xcodeproj/project.pbxproj

@@ -1049,7 +1049,6 @@
 		CD6A72B65D3B6076F4045C30 /* PHGPostHogConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = A6B891A6DA826E2461DBB40F /* PHGPostHogConfiguration.swift */; };
 		CDAE3A37D4DF136F9D07DB61 /* RoomChangeRolesScreenSection.swift in Sources */ = {isa = PBXBuildFile; fileRef = EAF710CB1C31F8938EAA3A7D /* RoomChangeRolesScreenSection.swift */; };
 		CDCA8A559E098503DDE29477 /* AttributedStringBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2A5C6FBF97B6EED3D4FA5EFF /* AttributedStringBuilder.swift */; };
-		CDE21091BB1272C088509F59 /* TemplateScreenTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B287771477D9A0D94AAC1CE /* TemplateScreenTests.swift */; };
 		CE1694C7BB93C3311524EF28 /* Untranslated.strings in Resources */ = {isa = PBXBuildFile; fileRef = D2F7194F440375338F8E2487 /* Untranslated.strings */; };
 		CE3B7FC34FB2C279AAA5EA01 /* AVMetadataMachineReadableCodeObject.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3339B1DDB1341E833D2555BC /* AVMetadataMachineReadableCodeObject.swift */; };
 		CE6F237360875D3D573FD0B2 /* RoomNotificationSettingsProxy.swift in Sources */ = {isa = PBXBuildFile; fileRef = AD6B522BD637845AB9570B10 /* RoomNotificationSettingsProxy.swift */; };
@@ -1956,7 +1955,6 @@
 		7B04BD3874D736127A8156B8 /* it */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = it; path = it.lproj/Localizable.strings; sourceTree = "<group>"; };
 		7B19B2BCC779ED934E0BBC2A /* AudioPlayer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioPlayer.swift; sourceTree = "<group>"; };
 		7B25F959A434BB9923A3223F /* ExpiringTaskRunner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExpiringTaskRunner.swift; sourceTree = "<group>"; };
-		7B287771477D9A0D94AAC1CE /* TemplateScreenTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemplateScreenTests.swift; sourceTree = "<group>"; };
 		7B3D16709ADD4F4BCC710B1E /* SecureBackupScreenModels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SecureBackupScreenModels.swift; sourceTree = "<group>"; };
 		7B849D2FF2CC12BA411A1651 /* CreateRoomModels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CreateRoomModels.swift; sourceTree = "<group>"; };
 		7B9FCA1CFD07B8CF9BD21266 /* FlowCoordinatorProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlowCoordinatorProtocol.swift; sourceTree = "<group>"; };
@@ -3796,7 +3794,6 @@
 		4B5DC42A1DB20ECEB0FF67CB /* Tests */ = {
 			isa = PBXGroup;
 			children = (
-				AD5FCF9340D670C526AD17E4 /* UI */,
 				73AB116809AE89292624CD8E /* Unit */,
 			);
 			path = Tests;
@@ -5206,14 +5203,6 @@
 			path = View;
 			sourceTree = "<group>";
 		};
-		AD5FCF9340D670C526AD17E4 /* UI */ = {
-			isa = PBXGroup;
-			children = (
-				7B287771477D9A0D94AAC1CE /* TemplateScreenTests.swift */,
-			);
-			path = UI;
-			sourceTree = "<group>";
-		};
 		B04B538A859CD012755DC19C /* NSE */ = {
 			isa = PBXGroup;
 			children = (
@@ -7858,7 +7847,6 @@
 				E9985DCD1B0D026D7E8BF809 /* ServerSelectionTests.swift in Sources */,
 				D5C2DA52162A978743A183F2 /* SessionVerificationTests.swift in Sources */,
 				F9788AE0B9EA19F1190ED14F /* StartChatScreenTests.swift in Sources */,
-				CDE21091BB1272C088509F59 /* TemplateScreenTests.swift in Sources */,
 				54AE8860D668AFD96E7E177B /* UITestsScreenIdentifier.swift in Sources */,
 				84EFCB95F9DA2979C8042B26 /* UITestsSignalling.swift in Sources */,
 				B22D857D1E8FCA6DD74A58E3 /* UserSessionScreenTests.swift in Sources */,

ElementX/Sources/UITests/UITestsAppCoordinator.swift

@@ -132,11 +132,6 @@ class MockScreen: Identifiable {
             flowCoordinator.start()
             retainedState.append(flowCoordinator)
             return nil
-        case .templateScreen:
-            let navigationStackCoordinator = NavigationStackCoordinator()
-            let coordinator = TemplateScreenCoordinator(parameters: .init())
-            navigationStackCoordinator.setRootCoordinator(coordinator)
-            return navigationStackCoordinator
         case .appLockFlow, .appLockFlowDisabled:
             // The tested coordinator is setup below in the alternate window.
             // Here we just return a blank screen to snapshot as the unlocked app.

ElementX/Sources/UITests/UITestsScreenIdentifier.swift

@@ -42,7 +42,6 @@ enum UITestsScreenIdentifier: String {
     case sessionVerification
     case startChat
     case startChatWithSearchResults
-    case templateScreen
     case userSessionScreen
     case userSessionScreenReply
     case autoUpdatingTimeline

Tools/Scripts/Templates/SimpleScreenExample/Tests/UI/TemplateScreenTests.swift

@@ -1,20 +0,0 @@
-//
-// Copyright 2022-2024 New Vector Ltd.
-//
-// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-// Please see LICENSE files in the repository root for full details.
-//
-
-import XCTest
-
-@MainActor
-class TemplateScreenUITests: XCTestCase {
-    func testScreen() async throws {
-        let app = Application.launch(.templateScreen)
-        
-        let title = app.staticTexts["Template title"]
-        XCTAssert(title.exists)
-
-        try await app.assertScreenshot()
-    }
-}

UITests/Sources/AuthenticationFlowCoordinatorTests.swift

@@ -98,7 +98,13 @@ class AuthenticationFlowCoordinatorUITests: XCTestCase {
         try await app.assertScreenshot()
     }
     
-    func testSelectingOIDCServer() {
+    // Disabled for now as the looping isn't 100% fool-proof and we have OIDC on the integration tests
+    // so this mock version doesn't really add anything to the tests as a whole.
+    func disabled_testSelectingOIDCServer() {
+        // Allow this test to run for longer to help with the loop whilst waiting to resolve the
+        // webcredentials for the Web Authentication Session (see below).
+        executionTimeAllowance = 300
+        
         // Given the authentication flow.
         let app = Application.launch(.authenticationFlow)
         
@@ -111,11 +117,31 @@ class AuthenticationFlowCoordinatorUITests: XCTestCase {
         // Server Selection: Clear the default, enter OIDC server and continue.
         app.textFields[A11yIdentifiers.changeServerScreen.server].clearAndTypeText("company.com\n", app: app)
         
-        // Server Confirmation: Tap continue button
-        app.buttons[A11yIdentifiers.serverConfirmationScreen.continue].tap()
-        
         let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
-        XCTAssertTrue(springboard.staticTexts["“ElementX” Wants to Use “company.com” to Sign In"].waitForExistence(timeout: 4),
+        let wasAlertText = springboard.staticTexts["“ElementX” Wants to Use “company.com” to Sign In"]
-                      "The web authentication prompt should be shown after selecting a homeserver with OIDC.")
+        
+        // On a fresh simulator the webcredentials association is sometimes slow to be resolved.
+        // This results in an error alert being shown instead of the Web Authentication Session alert.
+        // Keep looping on the Continue button for ~5 minutes until the Authentication Session is happy.
+        var remainingAttempts = 30
+        while !wasAlertText.exists {
+            // Server Confirmation: Tap continue button
+            app.buttons[A11yIdentifiers.serverConfirmationScreen.continue].tap()
+            
+            if wasAlertText.waitForExistence(timeout: 10) {
+                break
+            }
+            
+            remainingAttempts -= 1
+            if remainingAttempts <= 0 {
+                XCTFail("Failed to present the web authentication session.")
+            }
+            
+            if app.alerts.count > 0 {
+                app.alerts.firstMatch.buttons["OK"].tap()
+            }
+        }
+        
+        XCTAssertTrue(wasAlertText.exists, "The web authentication prompt should be shown after selecting a homeserver with OIDC.")
     }
 }

UITests/Sources/PollFormScreenTests.swift

@@ -17,15 +17,15 @@ class PollFormScreenUITests: XCTestCase {
     func testFilledPoll() async throws {
         let app = Application.launch(.createPoll)
         let questionTextField = app.textFields[A11yIdentifiers.pollFormScreen.question]
-        questionTextField.tapCenter()
+        questionTextField.tap(.center)
         questionTextField.typeText("Do you like polls?")
 
         let option1TextField = app.textFields[A11yIdentifiers.pollFormScreen.optionID(0)]
-        option1TextField.tapCenter()
+        option1TextField.tap(.center)
         option1TextField.typeText("Yes")
 
         let option2TextField = app.textFields[A11yIdentifiers.pollFormScreen.optionID(1)]
-        option2TextField.tapCenter()
+        option2TextField.tap(.center)
         option2TextField.typeText("No")
         
         // Dismiss the keyboard

UITests/Sources/UserSessionScreenTests.swift

@@ -23,7 +23,7 @@ class UserSessionScreenTests: XCTestCase {
         try await Task.sleep(for: .seconds(1))
         try await app.assertScreenshot(step: 2)
 
-        app.buttons[A11yIdentifiers.roomScreen.composerToolbar.openComposeOptions].tapCenter()
+        app.buttons[A11yIdentifiers.roomScreen.composerToolbar.openComposeOptions].tap(.center)
         try await app.assertScreenshot(step: 3)
     }
 

UITests/Sources/__Snapshots__/Application/serverSelection.testEmptyAddress-iPad-18-4-en-GB.png

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f41fca4268f52cffb50d6611762b3afe15074d56edd4dcc169dc48d4f4703aa
+oid sha256:09790b253032a3cba0bb2e3d0185239f314a9ca5261f222a8f206fec6c5b9803
-size 178019
+size 166204

UITests/Sources/__Snapshots__/Application/serverSelection.testEmptyAddress-iPhone-18-4-en-GB.png

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2622cc5aa6802a0b33ac982b327a1cfeb51f082509188316801526a5bd3b144f
+oid sha256:292478daac4382c9fb63766daa92e8277656ccabb46ed9780abe88ffc5c32a94
-size 185421
+size 163034

UITests/Sources/__Snapshots__/Application/serverSelection.testNormalState-iPad-18-4-en-GB.png

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03ec96c3117b93b21537a3e6a64f0b609722820d0fb967056adf2d4fabd6d0bb
+oid sha256:f1792419ba4926084b1862ed660385c9a288c30d5e819c041a7d203a133f0b7d
-size 104825
+size 92973

UITests/Sources/__Snapshots__/Application/serverSelection.testNormalState-iPhone-18-4-en-GB.png

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17c71fda8d1494f7fc67b51b304f77f53eba92ca19153aff3a4468f3fa6add80
+oid sha256:48bafe417ca3f7997f59592d8aa297faa7f0ddf6e0e9281513f835f77bba2670
-size 116342
+size 97386

UITests/Sources/__Snapshots__/Application/templateScreen.testScreen-iPad-18-4-en-GB.png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3a08af0ee7b494b7d1d3932141c743d1e0700e4cbb734fb66abd72696472bf12
-size 72787

UITests/Sources/__Snapshots__/Application/templateScreen.testScreen-iPhone-18-4-en-GB.png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:569e5ddc8cc188045103a62124f65f1ad550ee8b52b9d0f56b3692d00497dca3
-size 70422

UITests/SupportingFiles/target.yml

@@ -64,4 +64,3 @@ targets:
     - path: ../../ElementX/Sources/Other/Extensions/XCUIElement.swift
     - path: ../../ElementX/Sources/UITests/UITestsScreenIdentifier.swift
     - path: ../../ElementX/Sources/UITests/UITestsSignalling.swift
-    - path: ../../Tools/Scripts/Templates/SimpleScreenExample/Tests/UI

fastlane/Fastfile

@@ -105,17 +105,28 @@ lane :unit_tests do |options|
 end
 
 lane :ui_tests do |options|
-  create_simulator_if_necessary(
+  # We used to run these simultaneously on iPhone and iPad but it is *really* slow on GitHub runners.
-    name: "iPhone-18.4",
+  # Presumably because launching 2 simulators uses more memory than the runner has available.
-    type: "com.apple.CoreSimulator.SimDeviceType.iPhone-16",
-    runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4"
-  )
   
-  create_simulator_if_necessary(
+  if options[:device] == "iPhone"
-    name: "iPad-18.4",
+    device = "iPhone-18.4"
-    type: "com.apple.CoreSimulator.SimDeviceType.iPad-10th-generation",
+    
-    runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4"
+    create_simulator_if_necessary(
-  )
+      name: "iPhone-18.4",
+      type: "com.apple.CoreSimulator.SimDeviceType.iPhone-16",
+      runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4"
+    )
+  elsif options[:device] == "iPad"
+    device = "iPad-18.4"
+    
+    create_simulator_if_necessary(
+      name: "iPad-18.4",
+      type: "com.apple.CoreSimulator.SimDeviceType.iPad-10th-generation",
+      runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4"
+    )
+  else
+    UI.user_error!("Please supply a device argument as device:iPhone or device:iPad")
+  end
 
   if options[:test_name]
     test_to_run = ["UITests/#{options[:test_name]}"]
@@ -124,12 +135,12 @@ lane :ui_tests do |options|
   end
 
   reset_simulator = ENV.key?('CI')
-
+  
   run_tests(
     scheme: "UITests",
-    devices: ["iPhone-18.4", "iPad-18.4"],
+    device: device,
     ensure_devices_found: true,
-    prelaunch_simulator: true,
+    prelaunch_simulator: false,
     result_bundle: true,
     only_testing: test_to_run,
     number_of_retries: 3,