ehealthsolutions/element-x-ios
3
0
Fork 0
mirror of https://github.com/Novaloop-AG/element-x-ios.git synced 2026-11-02 21:33:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Disable broken UI Tests and remove navigation. (#577)

Browse Source 
Add locale to signalling service name.
Type strings character by character.
Add a default test timeout of 1 minute.

* Directly set UI Tests screens as the root screen.

Speeds up test runs by removing scrolling and searching for buttons.

* Parallelise test runs from Xcode (not fastlane).

* Disable signalling based tests.
This commit is contained in:
Doug
2023-02-14 16:25:24 +00:00
committed by GitHub
parent 944660800a
commit 21154c7eef
25 changed files with 107 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files
UITests/Sources/AnalyticsPromptUITests.swift
+1 -2
View File
@@ -20,8 +20,7 @@ import XCTest
class AnalyticsPromptUITests: XCTestCase {
/// Verify that the prompt is displayed correctly.
func testAnalyticsPrompt() {
let app = Application.launch()
app.goToScreenWithIdentifier(.analyticsPrompt)
let app = Application.launch(.analyticsPrompt)
app.assertScreenshot(.analyticsPrompt)
}
}
UITests/Sources/Application.swift
+15 -15
View File
@@ -18,9 +18,12 @@ import SnapshotTesting
import XCTest
struct Application {
static func launch() -> XCUIApplication {
static func launch(_ identifier: UITestsScreenIdentifier) -> XCUIApplication {
let app = XCUIApplication()
app.launchEnvironment = ["IS_RUNNING_UI_TESTS": "1"]
app.launchEnvironment = [
"IS_RUNNING_UI_TESTS": "1",
"UI_TESTS_SCREEN": identifier.rawValue
]
Bundle.elementFallbackLanguage = "en"
app.launch()
return app
@@ -28,22 +31,11 @@ struct Application {
}
extension XCUIApplication {
func goToScreenWithIdentifier(_ identifier: UITestScreenIdentifier) {
let button = buttons[identifier.rawValue]
let lastLabel = staticTexts["lastItem"]
while !button.isHittable, !lastLabel.isHittable {
swipeUp()
}
button.tap()
}
/// Assert screenshot for a screen with the given identifier. Does not fail if a screenshot is newly created.
/// - Parameter identifier: Identifier of the UI test screen
/// - Parameter step: An optional integer that can be used to take multiple snapshots per test identifier.
/// - Parameter insets: Optional insets with which to crop the image by.
func assertScreenshot(_ identifier: UITestScreenIdentifier, step: Int? = nil, insets: UIEdgeInsets? = nil) {
func assertScreenshot(_ identifier: UITestsScreenIdentifier, step: Int? = nil, insets: UIEdgeInsets? = nil) {
var snapshotName = identifier.rawValue
if let step {
snapshotName += "-\(step)"
@@ -72,7 +64,15 @@ extension XCUIApplication {
}
private var deviceName: String {
UIDevice.current.name
var name = UIDevice.current.name
// When running with parallel execution simulators are named "Clone 2 of iPhone 14" etc.
// Tidy this prefix out of the name to generate snapshots with the correct name.
if name.starts(with: "Clone "), let range = name.range(of: " of ") {
name = String(name[range.upperBound...])
}
return name
}
private var languageCode: String {
UITests/Sources/AuthenticationCoordinatorUITests.swift
+3 -6
View File
@@ -22,8 +22,7 @@ import XCTest
class AuthenticationCoordinatorUITests: XCTestCase {
func testLoginWithPassword() {
// Given the authentication flow.
let app = Application.launch()
app.goToScreenWithIdentifier(.authenticationFlow)
let app = Application.launch(.authenticationFlow)
// Splash Screen: Tap get started button
app.buttons["getStartedButton"].tap()
@@ -44,8 +43,7 @@ class AuthenticationCoordinatorUITests: XCTestCase {
func testLoginWithIncorrectPassword() async throws {
// Given the authentication flow.
let app = Application.launch()
app.goToScreenWithIdentifier(.authenticationFlow)
let app = Application.launch(.authenticationFlow)
// Splash Screen: Tap get started button
app.buttons["getStartedButton"].tap()
@@ -66,8 +64,7 @@ class AuthenticationCoordinatorUITests: XCTestCase {
func testSelectingOIDCServer() {
// Given the authentication flow.
let app = Application.launch()
app.goToScreenWithIdentifier(.authenticationFlow)
let app = Application.launch(.authenticationFlow)
// Splash Screen: Tap get started button
app.buttons["getStartedButton"].tap()
UITests/Sources/BugReportUITests.swift
+4 -8
View File
@@ -19,16 +19,14 @@ import XCTest
class BugReportUITests: XCTestCase {
func testInitialStateComponents() {
let app = Application.launch()
app.goToScreenWithIdentifier(.bugReport)
let app = Application.launch(.bugReport)
// Initial state without a screenshot attached.
app.assertScreenshot(.bugReport, step: 0)
}
func testToggleSendingLogs() {
let app = Application.launch()
app.goToScreenWithIdentifier(.bugReport)
let app = Application.launch(.bugReport)
app.switches["sendLogsToggle"].tap()
@@ -40,8 +38,7 @@ class BugReportUITests: XCTestCase {
}
func testReportText() {
let app = Application.launch()
app.goToScreenWithIdentifier(.bugReport)
let app = Application.launch(.bugReport)
// Type 4 characters and the send button should be disabled.
app.textViews["reportTextView"].clearAndTypeText("Text")
@@ -55,8 +52,7 @@ class BugReportUITests: XCTestCase {
}
func testInitialStateComponentsWithScreenshot() {
let app = Application.launch()
app.goToScreenWithIdentifier(.bugReportWithScreenshot)
let app = Application.launch(.bugReportWithScreenshot)
// Initial state with a screenshot attached.
XCTAssert(app.images["screenshotImage"].exists)
UITests/Sources/HomeScreenUITests.swift
+1 -2
View File
@@ -18,8 +18,7 @@ import XCTest
class HomeScreenUITests: XCTestCase {
func testInitialStateComponents() {
let app = Application.launch()
app.goToScreenWithIdentifier(.home)
let app = Application.launch(.home)
XCTAssert(app.navigationBars[ElementL10n.allChats].exists)
UITests/Sources/LoginScreenUITests.swift
+4 -9
View File
@@ -21,8 +21,7 @@ import XCTest
class LoginScreenUITests: XCTestCase {
func testMatrixDotOrg() {
// Given the initial login screen which defaults to matrix.org.
let app = Application.launch()
app.goToScreenWithIdentifier(.login)
let app = Application.launch(.login)
app.assertScreenshot(.login)
// When typing in a username and password.
@@ -35,8 +34,7 @@ class LoginScreenUITests: XCTestCase {
func testOIDC() {
// Given the initial login screen.
let app = Application.launch()
app.goToScreenWithIdentifier(.login)
let app = Application.launch(.login)
// When entering a username on a homeserver that only supports OIDC.
app.textFields["usernameTextField"].clearAndTypeText("@test:company.com\n")
@@ -47,13 +45,10 @@ class LoginScreenUITests: XCTestCase {
func testUnsupported() {
// Given the initial login screen.
let app = Application.launch()
app.goToScreenWithIdentifier(.login)
let app = Application.launch(.login)
// When entering a username on a homeserver with an unsupported flow.
let usernameTextField = app.textFields["usernameTextField"]
XCTAssertTrue(usernameTextField.waitForExistence(timeout: 5.0))
usernameTextField.clearAndTypeText("@test:server.net\n")
app.textFields["usernameTextField"].clearAndTypeText("@test:server.net\n")
// Then the screen should not allow login to continue.
app.assertScreenshot(.login, step: 2)
UITests/Sources/OnboardingUITests.swift
+2 -4
View File
@@ -19,15 +19,13 @@ import XCTest
@MainActor
class OnboardingUITests: XCTestCase {
func testInitialStateComponents() {
let app = Application.launch()
app.goToScreenWithIdentifier(.onboarding)
let app = Application.launch(.onboarding)
app.assertScreenshot(.onboarding)
}
// This test has been disabled for now as there is only a single page.
func disabled_testSwipingBetweenPages() {
let app = Application.launch()
app.goToScreenWithIdentifier(.onboarding)
let app = Application.launch(.onboarding)
// Given the splash screen in its initial state.
let page1TitleText = app.staticTexts[ElementL10n.ftueAuthCarouselSecureTitle]
UITests/Sources/RoomDetailsScreenUITests.swift
+2 -4
View File
@@ -19,16 +19,14 @@ import XCTest
class RoomDetailsScreenUITests: XCTestCase {
func testInitialStateComponents() {
let app = Application.launch()
app.goToScreenWithIdentifier(.roomDetailsScreen)
let app = Application.launch(.roomDetailsScreen)
XCTAssert(app.staticTexts["roomAvatarImage"].exists)
app.assertScreenshot(.roomDetailsScreen)
}
func testInitialStateComponentsWithRoomAvatar() {
let app = Application.launch()
app.goToScreenWithIdentifier(.roomDetailsScreenWithRoomAvatar)
let app = Application.launch(.roomDetailsScreenWithRoomAvatar)
XCTAssert(app.images["roomAvatarImage"].waitForExistence(timeout: 1))
app.assertScreenshot(.roomDetailsScreenWithRoomAvatar)
UITests/Sources/RoomMemberDetailsScreenUITests.swift
+1 -2
View File
@@ -19,8 +19,7 @@ import XCTest
class RoomMemberDetailsScreenUITests: XCTestCase {
func testInitialStateComponents() {
let app = Application.launch()
app.goToScreenWithIdentifier(.roomMemberDetailsScreen)
let app = Application.launch(.roomMemberDetailsScreen)
app.assertScreenshot(.roomMemberDetailsScreen)
}
UITests/Sources/RoomScreenUITests.swift
+21 -27
View File
@@ -19,9 +19,10 @@ import XCTest
@MainActor
class RoomScreenUITests: XCTestCase {
let connectionWaitDuration: Duration = .seconds(2)
func testPlainNoAvatar() {
let app = Application.launch()
app.goToScreenWithIdentifier(.roomPlainNoAvatar)
let app = Application.launch(.roomPlainNoAvatar)
XCTAssert(app.staticTexts["roomNameLabel"].exists)
XCTAssert(app.staticTexts["roomAvatarImage"].exists)
@@ -30,8 +31,7 @@ class RoomScreenUITests: XCTestCase {
}
func testEncryptedWithAvatar() {
let app = Application.launch()
app.goToScreenWithIdentifier(.roomEncryptedWithAvatar)
let app = Application.launch(.roomEncryptedWithAvatar)
XCTAssert(app.staticTexts["roomNameLabel"].exists)
XCTAssert(app.images["roomAvatarImage"].waitForExistence(timeout: 1))
@@ -40,21 +40,19 @@ class RoomScreenUITests: XCTestCase {
}
func testSmallTimelineLayout() {
let app = Application.launch()
app.goToScreenWithIdentifier(.roomSmallTimeline)
let app = Application.launch(.roomSmallTimeline)
// The messages should be bottom aligned.
app.assertScreenshot(.roomSmallTimeline)
}
func testSmallTimelineWithIncomingAndPagination() async throws {
func disabled_testSmallTimelineWithIncomingAndPagination() async throws {
let listener = try UITestsSignalling.Listener()
let app = Application.launch()
app.goToScreenWithIdentifier(.roomSmallTimelineIncomingAndSmallPagination)
let app = Application.launch(.roomSmallTimelineIncomingAndSmallPagination)
let connection = try await listener.connection()
try await Task.sleep(for: .seconds(1)) // Allow the connection to settle on CI/Intel...
try await Task.sleep(for: connectionWaitDuration) // Allow the connection to settle on CI/Intel...
defer { connection.disconnect() }
// When a back pagination occurs and an incoming message arrives.
@@ -65,14 +63,13 @@ class RoomScreenUITests: XCTestCase {
app.assertScreenshot(.roomSmallTimelineIncomingAndSmallPagination)
}
func testSmallTimelineWithLargePagination() async throws {
func disabled_testSmallTimelineWithLargePagination() async throws {
let listener = try UITestsSignalling.Listener()
let app = Application.launch()
app.goToScreenWithIdentifier(.roomSmallTimelineLargePagination)
let app = Application.launch(.roomSmallTimelineLargePagination)
let connection = try await listener.connection()
try await Task.sleep(for: .seconds(1)) // Allow the connection to settle on CI/Intel...
try await Task.sleep(for: connectionWaitDuration) // Allow the connection to settle on CI/Intel...
defer { connection.disconnect() }
// When a large back pagination occurs.
@@ -82,14 +79,13 @@ class RoomScreenUITests: XCTestCase {
app.assertScreenshot(.roomSmallTimelineLargePagination)
}
func testTimelineLayoutInMiddle() async throws {
func disabled_testTimelineLayoutInMiddle() async throws {
let listener = try UITestsSignalling.Listener()
let app = Application.launch()
app.goToScreenWithIdentifier(.roomLayoutMiddle)
let app = Application.launch(.roomLayoutMiddle)
let connection = try await listener.connection()
try await Task.sleep(for: .seconds(1)) // Allow the connection to settle on CI/Intel...
try await Task.sleep(for: connectionWaitDuration) // Allow the connection to settle on CI/Intel...
defer { connection.disconnect() }
// Given a timeline that is neither at the top nor the bottom.
@@ -116,14 +112,13 @@ class RoomScreenUITests: XCTestCase {
app.assertScreenshot(.roomLayoutMiddle, step: 1)
}
func testTimelineLayoutAtTop() async throws {
func disabled_testTimelineLayoutAtTop() async throws {
let listener = try UITestsSignalling.Listener()
let app = Application.launch()
app.goToScreenWithIdentifier(.roomLayoutTop)
let app = Application.launch(.roomLayoutTop)
let connection = try await listener.connection()
try await Task.sleep(for: .seconds(1)) // Allow the connection to settle on CI/Intel...
try await Task.sleep(for: connectionWaitDuration) // Allow the connection to settle on CI/Intel...
defer { connection.disconnect() }
// Given a timeline that is scrolled to the top.
@@ -140,14 +135,13 @@ class RoomScreenUITests: XCTestCase {
app.assertScreenshot(.roomLayoutTop, insets: cropped)
}
func testTimelineLayoutAtBottom() async throws {
func disabled_testTimelineLayoutAtBottom() async throws {
let listener = try UITestsSignalling.Listener()
let app = Application.launch()
app.goToScreenWithIdentifier(.roomLayoutBottom)
let app = Application.launch(.roomLayoutBottom)
let connection = try await listener.connection()
try await Task.sleep(for: .seconds(2)) // Allow the connection to settle on CI/Intel...
try await Task.sleep(for: connectionWaitDuration) // Allow the connection to settle on CI/Intel...
defer { connection.disconnect() }
// When an incoming message arrives.
@@ -168,7 +162,7 @@ class RoomScreenUITests: XCTestCase {
private func performOperation(_ operation: UITestsSignal, using connection: UITestsSignalling.Connection) async throws {
try await connection.send(operation)
guard try await connection.receive() == .success else { throw UITestsSignalError.unexpected }
try await Task.sleep(for: .milliseconds(500)) // Allow the timeline to update
try await Task.sleep(for: connectionWaitDuration) // Allow the timeline to update, and the connection to be ready
}
private func tapMessageComposer(in app: XCUIApplication) async throws {
UITests/Sources/ServerSelectionUITests.swift
+4 -8
View File
@@ -23,8 +23,7 @@ class ServerSelectionUITests: XCTestCase {
func testNormalState() async {
// Given the initial server selection screen as a modal.
let app = Application.launch()
app.goToScreenWithIdentifier(.serverSelection)
let app = Application.launch(.serverSelection)
// Then it should be configured for matrix.org
app.assertScreenshot(.serverSelection, step: 0)
@@ -34,8 +33,7 @@ class ServerSelectionUITests: XCTestCase {
func testEmptyAddress() async {
// Given the initial server selection screen as a modal.
let app = Application.launch()
app.goToScreenWithIdentifier(.serverSelection)
let app = Application.launch(.serverSelection)
// When clearing the server address text field.
app.textFields[textFieldIdentifier].tap()
@@ -49,8 +47,7 @@ class ServerSelectionUITests: XCTestCase {
func testInvalidAddress() {
// Given the initial server selection screen as a modal.
let app = Application.launch()
app.goToScreenWithIdentifier(.serverSelection)
let app = Application.launch(.serverSelection)
// When typing in an invalid homeserver
app.textFields[textFieldIdentifier].clearAndTypeText("thisisbad\n") // The tests only accept an address from LoginHomeserver.mockXYZ
@@ -63,8 +60,7 @@ class ServerSelectionUITests: XCTestCase {
func testNonModalPresentation() {
// Given the initial server selection screen pushed onto the stack.
let app = Application.launch()
app.goToScreenWithIdentifier(.serverSelectionNonModal)
let app = Application.launch(.serverSelectionNonModal)
// Then the screen should be tweaked slightly to reflect the change of navigation.
app.assertScreenshot(.serverSelectionNonModal)
UITests/Sources/SessionVerificationUITests.swift
+3 -6
View File
@@ -31,8 +31,7 @@ class SessionVerificationUITests: XCTestCase {
}
func testChallengeMatches() {
let app = Application.launch()
app.goToScreenWithIdentifier(.sessionVerification)
let app = Application.launch(.sessionVerification)
app.assertScreenshot(.sessionVerification, step: Step.initialState)
app.buttons["requestVerificationButton"].tap()
@@ -57,8 +56,7 @@ class SessionVerificationUITests: XCTestCase {
}
func testChallengeDoesNotMatch() {
let app = Application.launch()
app.goToScreenWithIdentifier(.sessionVerification)
let app = Application.launch(.sessionVerification)
app.assertScreenshot(.sessionVerification, step: Step.initialState)
app.buttons["requestVerificationButton"].tap()
@@ -80,8 +78,7 @@ class SessionVerificationUITests: XCTestCase {
}
func testSessionVerificationCancelation() {
let app = Application.launch()
app.goToScreenWithIdentifier(.sessionVerification)
let app = Application.launch(.sessionVerification)
app.assertScreenshot(.sessionVerification, step: Step.initialState)
app.buttons["requestVerificationButton"].tap()
UITests/Sources/SettingsScreenUITests.swift
+2 -3
View File
@@ -17,10 +17,9 @@
import ElementX
import XCTest
class SettingsUITests: XCTestCase {
class SettingsScreenUITests: XCTestCase {
func testInitialStateComponents() {
let app = Application.launch()
app.goToScreenWithIdentifier(.settings)
let app = Application.launch(.settings)
app.assertScreenshot(.settings)
}
}
UITests/Sources/SoftLogoutUITests.swift
+5 -19
View File
@@ -27,30 +27,16 @@ class SoftLogoutUITests: XCTestCase {
}
func testInitialState() {
app = Application.launch()
app.goToScreenWithIdentifier(.softLogout)
app = Application.launch(.softLogout)
XCTAssertTrue(app.staticTexts["titleLabel"].exists, "The title should be shown.")
XCTAssertTrue(app.staticTexts["messageLabel1"].exists, "The message 1 should be shown.")
XCTAssertTrue(app.staticTexts["clearDataTitleLabel"].exists, "The clear data title should be shown.")
XCTAssertTrue(app.staticTexts["clearDataMessageLabel"].exists, "The clear data message should be shown.")
let passwordTextField = app.secureTextFields["passwordTextField"]
XCTAssertTrue(passwordTextField.exists, "The password text field should be shown.")
XCTAssertTrue(passwordTextField.label.isEmpty, "The password text field text should be empty before text is input.")
XCTAssertEqual(passwordTextField.placeholderValue, ElementL10n.loginSignupPasswordHint, "The password text field should be showing the placeholder before text is input.")
let nextButton = app.buttons["nextButton"]
XCTAssertTrue(nextButton.exists, "The next button should be shown.")
XCTAssertFalse(nextButton.isEnabled, "The next button should be disabled before text is input.")
let forgotPasswordButton = app.buttons["forgotPasswordButton"]
XCTAssertTrue(forgotPasswordButton.exists, "The forgot password button should be shown.")
XCTAssertTrue(forgotPasswordButton.isEnabled, "The forgot password button should be enabled.")
let clearDataButton = app.buttons["clearDataButton"]
XCTAssertTrue(clearDataButton.exists, "The clear data button should be shown.")
XCTAssertTrue(clearDataButton.isEnabled, "The clear data button should be enabled.")
XCTAssertTrue(app.secureTextFields["passwordTextField"].exists, "The password text field should be shown.")
XCTAssertTrue(app.buttons["nextButton"].exists, "The next button should be shown.")
XCTAssertTrue(app.buttons["forgotPasswordButton"].exists, "The forgot password button should be shown.")
XCTAssertTrue(app.buttons["clearDataButton"].exists, "The clear data button should be shown.")
app.assertScreenshot(.softLogout)
}
UITests/Sources/UserSessionScreenTests.swift
+1 -2
View File
@@ -22,8 +22,7 @@ class UserSessionScreenTests: XCTestCase {
func testUserSessionFlows() async throws {
let roomName = "First room"
let app = Application.launch()
app.goToScreenWithIdentifier(.userSessionScreen)
let app = Application.launch(.userSessionScreen)
app.assertScreenshot(.userSessionScreen, step: 1)
UITests/SupportingFiles/UITests.xctestplan
+3
View File
@@ -30,6 +30,7 @@
}
]
},
"defaultTestExecutionTimeAllowance" : 60,
"environmentVariableEntries" : [
{
"key" : "SNAPSHOT_ARTIFACTS",
@@ -45,12 +46,14 @@
"identifier" : "0E28CD62691FDBC63147D5E3",
"name" : "UITests"
},
"testTimeoutsEnabled" : true,
"uiTestingScreenshotsLifetime" : "keepNever",
"undefinedBehaviorSanitizerEnabled" : true,
"userAttachmentLifetime" : "keepNever"
},
"testTargets" : [
{
"parallelizable" : true,
"target" : {
"containerPath" : "container:ElementX.xcodeproj",
"identifier" : "0E28CD62691FDBC63147D5E3",
UITests/SupportingFiles/target.yml
+1 -1
View File
@@ -58,7 +58,7 @@ targets:
- "**/__Snapshots__/**"
- path: ../SupportingFiles
- path: ../../Tools/Scripts/Templates/SimpleScreenExample/Tests/UI
- path: ../../ElementX/Sources/UITests/UITestScreenIdentifier.swift
- path: ../../ElementX/Sources/UITests/UITestsScreenIdentifier.swift
- path: ../../ElementX/Sources/UITests/UITestsSignalling.swift
- path: ../../ElementX/Sources/Generated/Strings.swift
- path: ../../ElementX/Sources/Generated/Strings+Untranslated.swift