#49, #123 - Setup UI integration and performance testing (#148)

IntegrationTests/Sources/Application.swift

@@ -0,0 +1,46 @@
+//
+//  Application.swift
+//  IntegrationTests
+//
+//  Created by Stefan Ceriu on 26/07/2022.
+//  Copyright © 2022 Element. All rights reserved.
+//
+
+import XCTest
+
+struct Application {
+    @discardableResult static func launch() -> XCUIApplication {
+        let app = XCUIApplication()
+        app.launch()
+        return app
+    }
+}
+
+extension XCUIApplication {
+    var homeserver: String {
+        guard let homeserver = ProcessInfo.processInfo.environment["INTEGRATION_TESTS_HOST"],
+              homeserver.count > 0 else {
+            return "default"
+        }
+        
+        return homeserver
+    }
+    
+    var username: String {
+        guard let username = ProcessInfo.processInfo.environment["INTEGRATION_TESTS_USERNAME"],
+              username.count > 0 else {
+            return "default"
+        }
+        
+        return username
+    }
+    
+    var password: String {
+        guard let password = ProcessInfo.processInfo.environment["INTEGRATION_TESTS_PASSWORD"],
+              password.count > 0 else {
+            return "default"
+        }
+        
+        return password
+    }
+}

IntegrationTests/Sources/ApplicationTests.swift

@@ -0,0 +1,29 @@
+//
+//  ApplicationTests.swift
+//  IntegrationTests
+//
+//  Created by Stefan Ceriu on 27/07/2022.
+//  Copyright © 2022 Element. All rights reserved.
+//
+
+import XCTest
+
+class ApplicationTests: XCTestCase {
+    func testLaunchPerformance() throws {
+        let parser = TestMeasurementParser()
+        
+        parser.capture(testCase: self) {
+            self.measure(metrics: [XCTApplicationLaunchMetric()]) {
+                Application.launch()
+            }
+        }
+        
+        guard let actualDuration = parser.valueForMetric(.appLaunch) else {
+            XCTFail("Couldn't retrieve app launch duration")
+            return
+        }
+        
+        let expectedDuration = 2.75
+        XCTAssertLessThanOrEqual(actualDuration, expectedDuration)
+    }
+}

IntegrationTests/Sources/LoginTests.swift

@@ -0,0 +1,81 @@
+//
+//  LoginTest.swift
+//  IntegrationTests
+//
+//  Created by Stefan Ceriu on 25/07/2022.
+//  Copyright © 2022 Element. All rights reserved.
+//
+
+import XCTest
+
+class LoginTests: XCTestCase {
+    let expectedDuration = 30.0
+    
+    func testLoginFlow() throws {
+        let parser = TestMeasurementParser()
+        parser.capture(testCase: self) {
+            self.measure(metrics: [XCTClockMetric()]) {
+                self.runLoginLogoutFlow()
+            }
+        }
+        
+        guard let actualDuration = parser.valueForMetric(.clockMonotonicTime) else {
+            XCTFail("Couldn't retrieve duration")
+            return
+        }
+        
+        XCTAssertLessThanOrEqual(actualDuration, expectedDuration)
+    }
+    
+    private func runLoginLogoutFlow() {
+        let app = Application.launch()
+        
+        let getStartedButton = app.buttons["Get started"]
+        
+        XCTAssertTrue(getStartedButton.waitForExistence(timeout: 5.0))
+        getStartedButton.tap()
+        
+        let editHomeserverButton = app.buttons["editServerButton"]
+        XCTAssertTrue(editHomeserverButton.waitForExistence(timeout: 5.0))
+        editHomeserverButton.tap()
+        
+        let homeserverTextField = app.textFields["addressTextField"]
+        XCTAssertTrue(homeserverTextField.waitForExistence(timeout: 5.0))
+        
+        homeserverTextField.clearAndTypeText(app.homeserver)
+        
+        let confirmButton = app.buttons["confirmButton"]
+        XCTAssertTrue(confirmButton.exists)
+        confirmButton.tap()
+        
+        let usernameTextField = app.textFields["usernameTextField"]
+        XCTAssertTrue(usernameTextField.exists)
+        
+        usernameTextField.tap()
+        usernameTextField.typeText(app.username)
+        
+        let passwordTextField = app.secureTextFields["passwordTextField"]
+        XCTAssertTrue(confirmButton.exists)
+        
+        passwordTextField.tap()
+        passwordTextField.typeText(app.password)
+        
+        let nextButton = app.buttons["nextButton"]
+        XCTAssertTrue(nextButton.exists)
+        XCTAssertTrue(nextButton.isEnabled)
+        
+        nextButton.tap()
+        
+        let profileButton = app.buttons["userDisplayNameView"]
+        XCTAssertTrue(profileButton.waitForExistence(timeout: expectedDuration))
+        profileButton.tap()
+        
+        let logoutButton = app.buttons["logoutButton"]
+        XCTAssertTrue(logoutButton.waitForExistence(timeout: 5.0))
+        logoutButton.tap()
+        
+        let logoutSheetButton = app.buttons["Sign out"].firstMatch
+        XCTAssertTrue(logoutSheetButton.waitForExistence(timeout: 5.0))
+        logoutSheetButton.tap()
+    }
+}

IntegrationTests/Sources/TestMeasurementParser.swift

@@ -0,0 +1,120 @@
+//
+//  TestMeasurementParser.swift
+//  IntegrationTests
+//
+//  Created by Stefan Ceriu on 28/07/2022.
+//  Copyright © 2022 Element. All rights reserved.
+//
+
+import Foundation
+import XCTest
+
+enum TestMeasurementParserMetric: String {
+    case appLaunch = "Duration (AppLaunch)"
+    case clockMonotonicTime = "Clock Monotonic Time"
+    case cpuTime = "CPU Time"
+    case memoryPeakPhysical = "Memory Peak Physical"
+    case memoryPhysical = "Memory Physical"
+    case cpuInstructionsRetired = "CPU Instructions Retired"
+    case diskLogicalWrites = "Disk Logical Writes"
+    case cpuCycles = "CPU Cycles"
+    
+    var regexValue: String {
+        switch self {
+        case .appLaunch:
+            return rawValue.replacingOccurrences(of: "(", with: "\\(").replacingOccurrences(of: ")", with: "\\)")
+        default:
+            return rawValue
+        }
+    }
+}
+
+/// This class is responsible for extracting XCTest measurement run results from stderr in lieu of an official API
+/// Heavily inspired by https://stackoverflow.com/questions/54814422/how-to-extract-performance-metrics-measured-by-measureblock-in-xctest
+class TestMeasurementParser {
+    private let pipe = Pipe()
+    private let regex: NSRegularExpression
+    private var results = [String: Double]()
+    
+    init() {
+        do {
+            let pattern = """
+            \\[(\
+            \(XCTestMeasurementParserMetric.appLaunch.regexValue)|\
+            \(XCTestMeasurementParserMetric.clockMonotonicTime.regexValue)|\
+            \(XCTestMeasurementParserMetric.cpuTime.regexValue)|\
+            \(XCTestMeasurementParserMetric.memoryPeakPhysical.regexValue)|\
+            \(XCTestMeasurementParserMetric.memoryPhysical.regexValue)|\
+            \(XCTestMeasurementParserMetric.cpuInstructionsRetired.regexValue)|\
+            \(XCTestMeasurementParserMetric.diskLogicalWrites.regexValue)|\
+            \(XCTestMeasurementParserMetric.cpuCycles.regexValue)\
+            ), \
+            (s|kB|kI|kC)\\] \
+            average: ([0-9\\.]*),
+            """
+            
+            regex = try NSRegularExpression(pattern: pattern,
+                                            options: .caseInsensitive)
+        } catch {
+            fatalError("Invalid regular expression")
+        }
+    }
+    
+    /// Running an XCTest measure block within this will enable the `XCTestMeasurementParser` to read resulting values from stderr and store them locally
+    /// - Parameters:
+    ///   - testCase: we need the test case so that we can add and wait for an expectation on it, allowing the capturing to finish
+    ///   - block: should contain the normal test steps and measure block
+    func capture(testCase: XCTestCase, block: @escaping () -> Void) {
+        results.removeAll()
+        
+        // Save original output
+        let original = dup(STDERR_FILENO)
+        
+        // Configure `stderr` so that it's not buffered
+        setvbuf(stderr, nil, _IONBF, 0)
+        
+        dup2(pipe.fileHandleForWriting.fileDescriptor, STDERR_FILENO)
+        
+        pipe.fileHandleForReading.readabilityHandler = { [weak self] handle in
+            guard let self = self else {
+                return
+            }
+            
+            let string = String(data: handle.availableData, encoding: .utf8) ?? "<Non-ascii data of size\(handle.availableData.count)>\n"
+            self.regex.matches(in: string, options: .reportCompletion, range: NSRange(location: 0, length: string.count)).forEach {
+                if let nameIndex = Range($0.range(at: 1), in: string),
+                   let averageIndex = Range($0.range(at: 3), in: string) {
+                    let name = String(string[nameIndex])
+                    let average = Double(string[averageIndex])
+                    
+                    self.results[name] = average
+                }
+            }
+            
+            // Print to stdout because stderr is piped
+            print("\(string)")
+        }
+        
+        block()
+        
+        // Revert
+        fflush(stderr)
+        dup2(original, STDERR_FILENO)
+        close(original)
+        
+        // Allow the parser to finish
+        let expectation = testCase.expectation(description: "Wait for the results to be gathered")
+        DispatchQueue.main.asyncAfter(deadline: .now() + 5.0) {
+            expectation.fulfill()
+        }
+        
+        testCase.waitForExpectations(timeout: 10.0)
+    }
+    
+    /// Retrieve the recorded average value for a particular metric
+    /// - Parameter metric: Needs to match one of the metrics passed in the XCTest measure metrics array
+    /// - Returns: The resulting average value
+    func valueForMetric(_ metric: XCTestMeasurementParserMetric) -> Double? {
+        results[metric.rawValue]
+    }
+}

IntegrationTests/Sources/XCUIElement.swift

@@ -0,0 +1,25 @@
+//
+//  XCUElement.swift
+//  ElementX
+//
+//  Created by Stefan Ceriu on 27/07/2022.
+//  Copyright © 2022 Element. All rights reserved.
+//
+
+import XCTest
+
+extension XCUIElement {
+    func clearAndTypeText(_ text: String) {
+        guard let stringValue = value as? String else {
+            XCTFail("Tried to clear and type text into a non string value")
+            return
+        }
+
+        tap()
+
+        let deleteString = String(repeating: XCUIKeyboardKey.delete.rawValue, count: stringValue.count)
+
+        typeText(deleteString)
+        typeText(text)
+    }
+}