Screenshot tests (#130)

* #9 Add snapshot testing library

* #9 Create script to boot test simulators

* #9 Create the UI test plan

* #9 Create shared schemes for test targets

* #9 Disable split view for UI tests

* #9 Fix fastlane dependencies

* #9 Add snapshot testing to the application

* #9 assert screenshots

* #9 fix swipe gestures on iPad

* #9 Fix accessing items in session verification screen

* #9 Workaround for flaky unit test

* #9 Specify scheme for alpha build

* #9 Add reference screenshots

* Update python script path and check assets for png check

* Update script path

* Use static timezone for simulator time

* Fix build after SwiftFormat

* Add changelog

* Upload failed screenshots artifact

* Always upload artifacts

* Update boot simulator script

* Update simulator overridden time

* Install pytz before tests

* Get time from Ruby script

* Disable SwiftUI animation when running UI tests

* Update screenshots after animation setting

* Include reference images in the artifact

* Update matching precision

* Update image matching precision & revert artifact content

* Include Xcode result in the artifact

* Update test output directory

* Disable gradient on splash screen for tests

* Tap next button explicitly

* Wait a bit before checking alert

* Wait 1 second

* Run SwiftFormat on project

* Ignore temporary screenshots

* Fix most of the PR remarks

* Fix conflicts

* Bump Python version to 3

* Update reference screenshots for authentication screens

* Update SwiftFormat

* Fix flakey session verification test.

* Update scheme.

Co-authored-by: Doug <douglase@element.io>

ElementX/Sources/AppDelegate.swift

@@ -18,7 +18,7 @@ import UIKit
 
 @main
 class AppDelegate: UIResponder, UIApplicationDelegate {
-    private lazy var appCoordinator: Coordinator = isRunningUITests ? UITestsAppCoordinator() : AppCoordinator()
+    private lazy var appCoordinator: Coordinator = Tests.isRunningUITests ? UITestsAppCoordinator() : AppCoordinator()
 
     func application(_ application: UIApplication, willFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil) -> Bool {
         //  use `en` as fallback language
@@ -28,7 +28,7 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
     }
     
     func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil) -> Bool {
-        if isRunningUnitTests {
+        if Tests.isRunningUnitTests {
             return true
         }
         
@@ -36,20 +36,4 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
         
         return true
     }
-    
-    private var isRunningUnitTests: Bool {
-        #if DEBUG
-        ProcessInfo.processInfo.environment["IS_RUNNING_UNIT_TESTS"] == "1"
-        #else
-        false
-        #endif
-    }
-    
-    private var isRunningUITests: Bool {
-        #if DEBUG
-        ProcessInfo.processInfo.environment["IS_RUNNING_UI_TESTS"] == "1"
-        #else
-        false
-        #endif
-    }
 }

ElementX/Sources/Other/SwiftUI/Animation/ElementAnimations.swift

@@ -0,0 +1,34 @@
+//
+//  ElementAnimations.swift
+//  ElementX
+//
+//  Created by Ismail on 8.07.2022.
+//  Copyright © 2022 Element. All rights reserved.
+//
+
+import Foundation
+import SwiftUI
+
+@available(iOS 13.0, macOS 10.15, tvOS 13.0, watchOS 6.0, *)
+public extension Animation {
+    /// Animation to be used to disable animations.
+    static let noAnimation: Animation = .linear(duration: 0)
+
+    /// `noAnimation` if running UI tests, otherwise `default` animation.
+    static var elementDefault: Animation {
+        Tests.isRunningUITests ? .noAnimation : .default
+    }
+}
+
+@available(iOS 13.0, macOS 10.15, tvOS 13.0, watchOS 6.0, *)
+/// Returns the result of recomputing the view's body with the provided
+/// animation.
+/// - Parameters:
+///   - animation: Animation
+///   - body: operations to be animated
+public func withElementAnimation<Result>(_ animation: Animation? = .default, _ body: () throws -> Result) rethrows -> Result {
+    if Tests.isRunningUITests {
+        return try withAnimation(.noAnimation, body)
+    }
+    return try withAnimation(animation, body)
+}

ElementX/Sources/Other/Tests.swift

@@ -0,0 +1,29 @@
+//
+//  Tests.swift
+//  ElementX
+//
+//  Created by Ismail on 29.07.2022.
+//  Copyright © 2022 Element. All rights reserved.
+//
+
+import Foundation
+
+public enum Tests {
+    /// Flag indicating whether the app is running the unit tests.
+    static var isRunningUnitTests: Bool {
+        #if DEBUG
+        ProcessInfo.processInfo.environment["IS_RUNNING_UNIT_TESTS"] == "1"
+        #else
+        false
+        #endif
+    }
+
+    /// Flag indicating whether the app is running the UI tests.
+    static var isRunningUITests: Bool {
+        #if DEBUG
+        ProcessInfo.processInfo.environment["IS_RUNNING_UI_TESTS"] == "1"
+        #else
+        false
+        #endif
+    }
+}

ElementX/Sources/Screens/Authentication/ServerSelection/ServerSelectionViewModel.swift

@@ -51,7 +51,7 @@ class ServerSelectionViewModel: ServerSelectionViewModelType, ServerSelectionVie
     func displayError(_ type: ServerSelectionErrorType) {
         switch type {
         case .footerMessage(let message):
-            withAnimation {
+            withElementAnimation {
                 state.footerErrorMessage = message
             }
         }
@@ -62,6 +62,6 @@ class ServerSelectionViewModel: ServerSelectionViewModelType, ServerSelectionVie
     /// Clear any errors shown in the text field footer.
     private func clearFooterError() {
         guard state.footerErrorMessage != nil else { return }
-        withAnimation { state.footerErrorMessage = nil }
+        withElementAnimation { state.footerErrorMessage = nil }
     }
 }

ElementX/Sources/Screens/HomeScreen/View/HomeScreen.swift

@@ -64,7 +64,7 @@ struct HomeScreen: View {
             Spacer()
         }
         .transition(.slide)
-        .animation(.default, value: context.viewState.showSessionVerificationBanner)
+        .animation(.elementDefault, value: context.viewState.showSessionVerificationBanner)
         .ignoresSafeArea(.all, edges: .bottom)
         .navigationBarTitleDisplayMode(.inline)
         .toolbar {
@@ -72,11 +72,11 @@ struct HomeScreen: View {
                 Button { context.send(viewAction: .tapUserAvatar) } label: {
                     HStack {
                         userAvatarImage
-                            .animation(.default, value: context.viewState.userAvatar)
+                            .animation(.elementDefault, value: context.viewState.userAvatar)
                             .transition(.opacity)
 
                         userDisplayNameView
-                            .animation(.default, value: context.viewState.userDisplayName)
+                            .animation(.elementDefault, value: context.viewState.userDisplayName)
                             .transition(.opacity)
                     }
                 }
@@ -147,7 +147,7 @@ struct RoomCell: View {
                     }
                 }
             }
-            .animation(.default, value: room)
+            .animation(.elementDefault, value: room)
             .frame(minHeight: 60.0)
             .task {
                 context.send(viewAction: .loadRoomData(roomIdentifier: room.id))

ElementX/Sources/Screens/RoomScreen/View/MessageComposer.swift

@@ -36,7 +36,7 @@ struct MessageComposer: View {
             .padding(.bottom, 6.0)
             .disabled(disabled)
             .opacity(disabled ? 0.5 : 1.0)
-            .animation(.default, value: disabled)
+            .animation(.elementDefault, value: disabled)
             .keyboardShortcut(.return, modifiers: [.command])
         }
     }

ElementX/Sources/Screens/RoomScreen/View/MessageComposerTextField.swift

@@ -57,7 +57,7 @@ struct MessageComposerTextField: View {
             .background(placeholderView, alignment: .topLeading)
             .clipShape(rect)
             .overlay(rect.stroke(borderColor, lineWidth: borderWidth))
-            .animation(.default, value: isEditing)
+            .animation(.elementDefault, value: isEditing)
     }
     
     @ViewBuilder

ElementX/Sources/Screens/RoomScreen/View/Timeline/ImageRoomTimelineView.swift

@@ -42,7 +42,7 @@ struct ImageRoomTimelineView: View {
                 }
             }
             .id(timelineItem.id)
-            .animation(.default, value: timelineItem.image)
+            .animation(.elementDefault, value: timelineItem.image)
             .frame(maxHeight: 1000.0)
         } else {
             TimelineStyler(timelineItem: timelineItem) {

ElementX/Sources/Screens/RoomScreen/View/TimelineItemList.swift

@@ -37,7 +37,7 @@ struct TimelineItemList: View {
                     Spacer()
                     ProgressView()
                         .opacity(context.viewState.isBackPaginating ? 1.0 : 0.0)
-                        .animation(.default, value: context.viewState.isBackPaginating)
+                        .animation(.elementDefault, value: context.viewState.isBackPaginating)
                     Spacer()
                 }
                 .listRowBackground(Color.clear)

ElementX/Sources/Screens/RoomScreen/View/TimelineSenderAvatarView.swift

@@ -40,6 +40,6 @@ struct TimelineSenderAvatarView: View {
                 .stroke(Color.element.background, lineWidth: 2)
         )
 
-        .animation(.default, value: timelineItem.senderAvatar)
+        .animation(.elementDefault, value: timelineItem.senderAvatar)
     }
 }

ElementX/Sources/Screens/RoomScreen/View/TimelineView.swift

@@ -47,7 +47,7 @@ struct TimelineView: View {
             scollToBottomButtonVisible = !visible
         })
         .opacity(scollToBottomButtonVisible ? 1.0 : 0.0)
-        .animation(.default, value: scollToBottomButtonVisible)
+        .animation(.elementDefault, value: scollToBottomButtonVisible)
     }
 }
 

ElementX/Sources/Screens/SessionVerification/SessionVerificationViewModel.swift

@@ -50,6 +50,11 @@ class SessionVerificationViewModel: SessionVerificationViewModelType, SessionVer
                 
                 switch callback {
                 case .receivedVerificationData(let emojis):
+                    guard self.stateMachine.state == .requestingVerification else {
+                        MXLog.warning("[SessionVerificationViewModel] callbacks: Ignoring receivedVerificationData due to invalid state.")
+                        return
+                    }
+                    
                     self.stateMachine.processEvent(.didReceiveChallenge(emojis: emojis))
                 case .finished:
                     self.stateMachine.processEvent(.didAcceptChallenge)

ElementX/Sources/Screens/SplashScreen/SplashScreenModels.swift

@@ -53,6 +53,9 @@ struct SplashScreenViewState: BindableState {
     
     /// The background gradient for all 4 pages and the hidden page at the start of the carousel.
     var backgroundGradient: Gradient {
+        if Tests.isRunningUITests {
+            return Gradient(colors: [.white])
+        }
         // Include the extra stop for the hidden page at the start of the carousel.
         // (The last color is the right-hand stop, but we need the left-hand stop,
         // so take the last but one color from the array).

ElementX/Sources/Screens/SplashScreen/View/SplashScreen.swift

@@ -123,11 +123,11 @@ struct SplashScreen: View {
             if context.pageIndex == pageCount - 1 {
                 showHiddenPage()
                 
-                withAnimation(.easeInOut(duration: 0.7)) {
+                withElementAnimation(.easeInOut(duration: 0.7)) {
                     showNextPage()
                 }
             } else {
-                withAnimation(.easeInOut(duration: 0.7)) {
+                withElementAnimation(.easeInOut(duration: 0.7)) {
                     showNextPage()
                 }
             }
@@ -185,7 +185,7 @@ struct SplashScreen: View {
         stopTimer()
         
         // Animate the change over a few frames to smooth out any stuttering.
-        withAnimation(.linear(duration: 0.05)) {
+        withElementAnimation(.linear(duration: 0.05)) {
             dragOffset = isLeftToRight ? drag.translation.width : -drag.translation.width
         }
     }
@@ -195,11 +195,11 @@ struct SplashScreen: View {
     private func handleDragGestureEnded(_ drag: DragGesture.Value, viewSize: CGSize) {
         guard shouldSwipeForTranslation(drag.predictedEndTranslation.width) else {
             // Reset the offset just in case.
-            withAnimation { dragOffset = 0 }
+            withElementAnimation { dragOffset = 0 }
             return
         }
         
-        withAnimation(.easeInOut(duration: 0.2)) {
+        withElementAnimation(.easeInOut(duration: 0.2)) {
             if drag.predictedEndTranslation.width < -viewSize.width / 2 {
                 showNextPage()
             } else if drag.predictedEndTranslation.width > viewSize.width / 2 {

ElementX/Sources/Services/BugReport/ScreenshotDetector.swift

@@ -89,7 +89,7 @@ class ScreenshotDetector {
     }
 }
 
-private extension PHAsset {
+extension PHAsset {
     static func fetchLastScreenshot() -> PHAsset? {
         let options = PHFetchOptions()
 

ElementX/Sources/UITestsRootView.swift

@@ -31,5 +31,6 @@ struct UITestsRootView: View {
             .listStyle(.plain)
         }
         .navigationTitle("Screens")
+        .navigationViewStyle(.stack)
     }
 }