Automatic Accessibility Audits on previews part 1 (#4322)

* setup target

* accessibility tests implementation

* removed ipad check

* pr improvements

* updated project
This commit is contained in:
Mauro
2025-07-15 14:40:16 +02:00
committed by GitHub
parent 74952659e8
commit c1b94a2d9f
18 changed files with 951 additions and 8 deletions
@@ -0,0 +1,155 @@
//
// Copyright 2025 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
// Please see LICENSE files in the repository root for full details.
//
import Combine
import SwiftUI
class AccessibilityTestsAppCoordinator: AppCoordinatorProtocol {
var windowManager: any SecureWindowManagerProtocol
func handleDeepLink(_ url: URL, isExternalURL: Bool) -> Bool {
fatalError("Not implemented")
}
func handlePotentialPhishingAttempt(url: URL, openURLAction: @escaping (URL) -> Void) -> Bool {
fatalError("Not implemented")
}
func handleUserActivity(_ userActivity: NSUserActivity) {
fatalError("Not implemented")
}
private let previewsWrapper: PreviewsWrapper
private var cancellables = Set<AnyCancellable>()
init(appDelegate: AppDelegate) {
windowManager = WindowManager(appDelegate: appDelegate)
// disabling View animations
UIView.setAnimationsEnabled(false)
MXLog.configure(currentTarget: "accessibility-tests")
ServiceLocator.shared.register(userIndicatorController: UserIndicatorController())
AppSettings.configureWithSuiteName("io.element.elementx.accessibilitytests")
AppSettings.resetAllSettings()
ServiceLocator.shared.register(appSettings: AppSettings())
ServiceLocator.shared.register(bugReportService: BugReportServiceMock(.init()))
let analyticsClient = AnalyticsClientMock()
analyticsClient.isRunning = false
ServiceLocator.shared.register(analytics: AnalyticsService(client: analyticsClient,
appSettings: ServiceLocator.shared.settings))
guard let name = ProcessInfo.accessibilityViewID,
let previewType = TestablePreviewsDictionary.dictionary[name] else {
fatalError("Unable to launch with unknown screen.")
}
previewsWrapper = .init(name: name, previews: previewType._allPreviews)
setupSignalling()
}
func toPresentable() -> AnyView {
AnyView(PreviewsWrapperView(wrapper: previewsWrapper))
}
private func setupSignalling() {
do {
let client = try UITestsSignalling.Client(mode: .app)
client.signals.sink { [weak self] signal in
guard let self else { return }
switch signal {
case .accessibilityAudit(let auditSignal):
switch auditSignal {
case .nextPreview:
Task { [weak self] in
guard let self else { return }
await previewsWrapper.updateCurrentIndex()
do {
guard !previewsWrapper.isDone else {
try client.send(.accessibilityAudit(.noMorePreviews))
return
}
try client.send(.accessibilityAudit(.nextPreviewReady(name: previewsWrapper.previewName)))
} catch {
fatalError("Failed sending signal: \(signal)")
}
}
default:
break
}
default:
break
}
}
.store(in: &cancellables)
} catch {
fatalError("Unable to start client signalling")
}
}
}
struct PreviewsWrapperView: View {
let wrapper: PreviewsWrapper
var body: some View {
if wrapper.currentIndex < 0 || wrapper.isDone {
EmptyView()
} else {
wrapper.currentPreview.content
.id(wrapper.previewName)
}
}
}
@Observable final class PreviewsWrapper {
private let name: String
private let previews: [_Preview]
private(set) var currentIndex = -1
var currentPreview: _Preview { previews[currentIndex] }
private(set) var isDone = false
var previewName: String {
"\(name)-\(currentPreview.displayName ?? String(currentIndex))"
}
init(name: String, previews: [_Preview]) {
self.name = name
self.previews = previews
}
@MainActor
func updateCurrentIndex() async {
let newIndex = currentIndex + 1
guard newIndex < previews.count else {
isDone = true
return
}
let newPreview = previews[newIndex]
var fulfillmentSource: SnapshotFulfillmentPreferenceKey.Source?
let preferenceReadingView = newPreview.content.onPreferenceChange(SnapshotFulfillmentPreferenceKey.self) { fulfillmentSource = $0?.source }
// Render an image of the view in order to trigger the preference updates to occur.
let imageRenderer = ImageRenderer(content: preferenceReadingView)
_ = imageRenderer.uiImage
switch fulfillmentSource {
case .publisher(let publisher):
_ = await publisher.values.first { $0 == true }
case .stream(let stream):
_ = await stream.first { $0 == true }
case .none:
break
}
currentIndex = newIndex
}
}
@@ -19,6 +19,8 @@ struct Application: App {
appCoordinator = UITestsAppCoordinator(appDelegate: appDelegate)
} else if ProcessInfo.isRunningUnitTests {
appCoordinator = UnitTestsAppCoordinator(appDelegate: appDelegate)
} else if ProcessInfo.isRunningAccessibilityTests {
appCoordinator = AccessibilityTestsAppCoordinator(appDelegate: appDelegate)
} else {
appCoordinator = AppCoordinator(appDelegate: appDelegate)
}
@@ -33,10 +33,18 @@ extension ProcessInfo {
false
#endif
}
static var isRunningAccessibilityTests: Bool {
#if DEBUG
processInfo.environment["ACCESSIBILITY_VIEW"] != nil
#else
false
#endif
}
/// Flag indicating whether the app is running the UI tests or unit tests.
static var isRunningTests: Bool {
isRunningUITests || isRunningUnitTests || isRunningIntegrationTests
isRunningUITests || isRunningUnitTests || isRunningIntegrationTests || isRunningAccessibilityTests
}
/// The identifier of the screen to be loaded when running UI tests.
@@ -48,6 +56,15 @@ extension ProcessInfo {
#endif
}
/// The identifier of the preview that will be accessibility tested
static var accessibilityViewID: String? {
#if DEBUG
processInfo.environment["ACCESSIBILITY_VIEW"]
#else
nil
#endif
}
static var shouldDisableTimelineAccessibility: Bool {
guard isRunningUITests else {
return false
@@ -0,0 +1,205 @@
// Generated using Sourcery 2.2.7 — https://github.com/krzysztofzablocki/Sourcery
// DO NOT EDIT
// swiftlint:disable all
// swiftformat:disable all
import SwiftUI
enum TestablePreviewsDictionary {
static let dictionary: [String: any PreviewProvider.Type] = [
"AdvancedSettingsScreen_Previews" : AdvancedSettingsScreen_Previews.self,
"AnalyticsPromptScreen_Previews" : AnalyticsPromptScreen_Previews.self,
"AnalyticsSettingsScreen_Previews" : AnalyticsSettingsScreen_Previews.self,
"AppLockScreen_Previews" : AppLockScreen_Previews.self,
"AppLockSetupBiometricsScreen_Previews" : AppLockSetupBiometricsScreen_Previews.self,
"AppLockSetupPINScreen_Previews" : AppLockSetupPINScreen_Previews.self,
"AppLockSetupSettingsScreen_Previews" : AppLockSetupSettingsScreen_Previews.self,
"AudioMediaEventsTimelineView_Previews" : AudioMediaEventsTimelineView_Previews.self,
"AudioRoomTimelineView_Previews" : AudioRoomTimelineView_Previews.self,
"AuthenticationStartScreen_Previews" : AuthenticationStartScreen_Previews.self,
"AvatarHeaderView_Previews" : AvatarHeaderView_Previews.self,
"BadgeLabel_Previews" : BadgeLabel_Previews.self,
"BigIcon_Previews" : BigIcon_Previews.self,
"BlockedUsersScreen_Previews" : BlockedUsersScreen_Previews.self,
"BugReportScreen_Previews" : BugReportScreen_Previews.self,
"CallInviteRoomTimelineView_Previews" : CallInviteRoomTimelineView_Previews.self,
"CallNotificationRoomTimelineView_Previews" : CallNotificationRoomTimelineView_Previews.self,
"CollapsibleRoomTimelineView_Previews" : CollapsibleRoomTimelineView_Previews.self,
"CompletionSuggestion_Previews" : CompletionSuggestion_Previews.self,
"ComposerToolbar_Previews" : ComposerToolbar_Previews.self,
"CreateRoom_Previews" : CreateRoom_Previews.self,
"DeactivateAccountScreen_Previews" : DeactivateAccountScreen_Previews.self,
"DeclineAndBlockScreen_Previews" : DeclineAndBlockScreen_Previews.self,
"EditRoomAddressScreen_Previews" : EditRoomAddressScreen_Previews.self,
"ElementTextFieldStyle_Previews" : ElementTextFieldStyle_Previews.self,
"EmojiPickerScreenHeaderView_Previews" : EmojiPickerScreenHeaderView_Previews.self,
"EmojiPickerScreen_Previews" : EmojiPickerScreen_Previews.self,
"EmoteRoomTimelineView_Previews" : EmoteRoomTimelineView_Previews.self,
"EncryptedRoomTimelineView_Previews" : EncryptedRoomTimelineView_Previews.self,
"EncryptionResetPasswordScreen_Previews" : EncryptionResetPasswordScreen_Previews.self,
"EncryptionResetScreen_Previews" : EncryptionResetScreen_Previews.self,
"EstimatedWaveformView_Previews" : EstimatedWaveformView_Previews.self,
"FileMediaEventsTimelineView_Previews" : FileMediaEventsTimelineView_Previews.self,
"FileRoomTimelineView_Previews" : FileRoomTimelineView_Previews.self,
"FormButtonStyles_Previews" : FormButtonStyles_Previews.self,
"FormattedBodyText_Previews" : FormattedBodyText_Previews.self,
"FormattingToolbar_Previews" : FormattingToolbar_Previews.self,
"FullscreenDialog_Previews" : FullscreenDialog_Previews.self,
"GlobalSearchScreenListRow_Previews" : GlobalSearchScreenListRow_Previews.self,
"GlobalSearchScreen_Previews" : GlobalSearchScreen_Previews.self,
"HighlightedTimelineItemModifier_Previews" : HighlightedTimelineItemModifier_Previews.self,
"HomeScreenEmptyStateView_Previews" : HomeScreenEmptyStateView_Previews.self,
"HomeScreenInviteCell_Previews" : HomeScreenInviteCell_Previews.self,
"HomeScreenKnockedCell_Previews" : HomeScreenKnockedCell_Previews.self,
"HomeScreenRecoveryKeyConfirmationBanner_Previews" : HomeScreenRecoveryKeyConfirmationBanner_Previews.self,
"HomeScreenRoomCell_Previews" : HomeScreenRoomCell_Previews.self,
"HomeScreen_Previews" : HomeScreen_Previews.self,
"IdentityConfirmationScreen_Previews" : IdentityConfirmationScreen_Previews.self,
"IdentityConfirmedScreen_Previews" : IdentityConfirmedScreen_Previews.self,
"ImageMediaEventsTimelineView_Previews" : ImageMediaEventsTimelineView_Previews.self,
"ImageRoomTimelineView_Previews" : ImageRoomTimelineView_Previews.self,
"InviteUsersScreenSelectedItem_Previews" : InviteUsersScreenSelectedItem_Previews.self,
"InviteUsersScreen_Previews" : InviteUsersScreen_Previews.self,
"JoinRoomByAddressView_Previews" : JoinRoomByAddressView_Previews.self,
"JoinRoomScreen_Previews" : JoinRoomScreen_Previews.self,
"KnockRequestCell_Previews" : KnockRequestCell_Previews.self,
"KnockRequestsBannerView_Previews" : KnockRequestsBannerView_Previews.self,
"KnockRequestsListEmptyStateView_Previews" : KnockRequestsListEmptyStateView_Previews.self,
"KnockRequestsListScreen_Previews" : KnockRequestsListScreen_Previews.self,
"LegalInformationScreen_Previews" : LegalInformationScreen_Previews.self,
"LoadableImage_Previews" : LoadableImage_Previews.self,
"LocationMarkerView_Previews" : LocationMarkerView_Previews.self,
"LocationRoomTimelineView_Previews" : LocationRoomTimelineView_Previews.self,
"LoginScreen_Previews" : LoginScreen_Previews.self,
"LongPressWithFeedback_Previews" : LongPressWithFeedback_Previews.self,
"ManageRoomMemberSheetView_Previews" : ManageRoomMemberSheetView_Previews.self,
"MapLibreStaticMapView_Previews" : MapLibreStaticMapView_Previews.self,
"MatrixUserPermalink_Previews" : MatrixUserPermalink_Previews.self,
"MediaEventsTimelineScreen_Previews" : MediaEventsTimelineScreen_Previews.self,
"MediaUploadPreviewScreen_Previews" : MediaUploadPreviewScreen_Previews.self,
"MentionSuggestionItemView_Previews" : MentionSuggestionItemView_Previews.self,
"MessageComposerTextField_Previews" : MessageComposerTextField_Previews.self,
"MessageComposer_Previews" : MessageComposer_Previews.self,
"MessageForwardingScreen_Previews" : MessageForwardingScreen_Previews.self,
"MessageText_Previews" : MessageText_Previews.self,
"NoticeRoomTimelineView_Previews" : NoticeRoomTimelineView_Previews.self,
"NotificationPermissionsScreen_Previews" : NotificationPermissionsScreen_Previews.self,
"NotificationSettingsEditScreenRoomCell_Previews" : NotificationSettingsEditScreenRoomCell_Previews.self,
"NotificationSettingsEditScreen_Previews" : NotificationSettingsEditScreen_Previews.self,
"NotificationSettingsScreen_Previews" : NotificationSettingsScreen_Previews.self,
"PINTextField_Previews" : PINTextField_Previews.self,
"PaginationIndicatorRoomTimelineView_Previews" : PaginationIndicatorRoomTimelineView_Previews.self,
"PillViewOnBubble_Previews" : PillViewOnBubble_Previews.self,
"PillView_Previews" : PillView_Previews.self,
"PinnedEventsTimelineScreen_Previews" : PinnedEventsTimelineScreen_Previews.self,
"PinnedItemsBannerView_Previews" : PinnedItemsBannerView_Previews.self,
"PinnedItemsIndicatorView_Previews" : PinnedItemsIndicatorView_Previews.self,
"PlaceholderAvatarImage_Previews" : PlaceholderAvatarImage_Previews.self,
"PlaceholderScreen_Previews" : PlaceholderScreen_Previews.self,
"PollFormScreen_Previews" : PollFormScreen_Previews.self,
"PollOptionView_Previews" : PollOptionView_Previews.self,
"PollRoomTimelineView_Previews" : PollRoomTimelineView_Previews.self,
"PollView_Previews" : PollView_Previews.self,
"QRCodeLoginScreen_Previews" : QRCodeLoginScreen_Previews.self,
"ReactionsSummaryView_Previews" : ReactionsSummaryView_Previews.self,
"ReadMarkerRoomTimelineView_Previews" : ReadMarkerRoomTimelineView_Previews.self,
"ReadReceiptCell_Previews" : ReadReceiptCell_Previews.self,
"ReadReceiptsSummaryView_Previews" : ReadReceiptsSummaryView_Previews.self,
"RedactedRoomTimelineView_Previews" : RedactedRoomTimelineView_Previews.self,
"ReportContentScreen_Previews" : ReportContentScreen_Previews.self,
"ReportRoomScreen_Previews" : ReportRoomScreen_Previews.self,
"ResolveVerifiedUserSendFailureScreen_Previews" : ResolveVerifiedUserSendFailureScreen_Previews.self,
"RoomAttachmentPicker_Previews" : RoomAttachmentPicker_Previews.self,
"RoomAvatarImage_Previews" : RoomAvatarImage_Previews.self,
"RoomChangePermissionsScreen_Previews" : RoomChangePermissionsScreen_Previews.self,
"RoomChangeRolesScreenRow_Previews" : RoomChangeRolesScreenRow_Previews.self,
"RoomChangeRolesScreenSelectedItem_Previews" : RoomChangeRolesScreenSelectedItem_Previews.self,
"RoomChangeRolesScreen_Previews" : RoomChangeRolesScreen_Previews.self,
"RoomDetailsEditScreen_Previews" : RoomDetailsEditScreen_Previews.self,
"RoomDetailsScreen_Previews" : RoomDetailsScreen_Previews.self,
"RoomDirectorySearchCell_Previews" : RoomDirectorySearchCell_Previews.self,
"RoomDirectorySearchScreen_Previews" : RoomDirectorySearchScreen_Previews.self,
"RoomHeaderView_Previews" : RoomHeaderView_Previews.self,
"RoomInviterLabel_Previews" : RoomInviterLabel_Previews.self,
"RoomListFilterView_Previews" : RoomListFilterView_Previews.self,
"RoomListFiltersEmptyStateView_Previews" : RoomListFiltersEmptyStateView_Previews.self,
"RoomListFiltersView_Previews" : RoomListFiltersView_Previews.self,
"RoomMemberDetailsScreen_Previews" : RoomMemberDetailsScreen_Previews.self,
"RoomMembersListMemberCell_Previews" : RoomMembersListMemberCell_Previews.self,
"RoomMembersListScreen_Previews" : RoomMembersListScreen_Previews.self,
"RoomNotificationSettingsCustomSectionView_Previews" : RoomNotificationSettingsCustomSectionView_Previews.self,
"RoomNotificationSettingsScreen_Previews" : RoomNotificationSettingsScreen_Previews.self,
"RoomNotificationSettingsUserDefinedScreen_Previews" : RoomNotificationSettingsUserDefinedScreen_Previews.self,
"RoomPollsHistoryScreen_Previews" : RoomPollsHistoryScreen_Previews.self,
"RoomRolesAndPermissionsScreen_Previews" : RoomRolesAndPermissionsScreen_Previews.self,
"RoomScreenFooterView_Previews" : RoomScreenFooterView_Previews.self,
"RoomScreen_Previews" : RoomScreen_Previews.self,
"RoomSelectionScreen_Previews" : RoomSelectionScreen_Previews.self,
"SFNumberedListView_Previews" : SFNumberedListView_Previews.self,
"SecureBackupKeyBackupScreen_Previews" : SecureBackupKeyBackupScreen_Previews.self,
"SecureBackupLogoutConfirmationScreen_Previews" : SecureBackupLogoutConfirmationScreen_Previews.self,
"SecureBackupRecoveryKeyScreen_Previews" : SecureBackupRecoveryKeyScreen_Previews.self,
"SecureBackupScreen_Previews" : SecureBackupScreen_Previews.self,
"SecurityAndPrivacyScreen_Previews" : SecurityAndPrivacyScreen_Previews.self,
"SendInviteConfirmationView_Previews" : SendInviteConfirmationView_Previews.self,
"SeparatorMediaEventsTimelineView_Previews" : SeparatorMediaEventsTimelineView_Previews.self,
"SeparatorRoomTimelineView_Previews" : SeparatorRoomTimelineView_Previews.self,
"ServerConfirmationScreen_Previews" : ServerConfirmationScreen_Previews.self,
"ServerSelection_Previews" : ServerSelection_Previews.self,
"SessionVerificationRequestDetailsView_Previews" : SessionVerificationRequestDetailsView_Previews.self,
"SessionVerification_Previews" : SessionVerification_Previews.self,
"SettingsScreen_Previews" : SettingsScreen_Previews.self,
"ShimmerOverlay_Previews" : ShimmerOverlay_Previews.self,
"SoftLogoutScreen_Previews" : SoftLogoutScreen_Previews.self,
"SplashScreen_Previews" : SplashScreen_Previews.self,
"StackedAvatarsView_Previews" : StackedAvatarsView_Previews.self,
"StartChatScreen_Previews" : StartChatScreen_Previews.self,
"StateRoomTimelineView_Previews" : StateRoomTimelineView_Previews.self,
"StaticLocationScreenViewer_Previews" : StaticLocationScreenViewer_Previews.self,
"StickerRoomTimelineView_Previews" : StickerRoomTimelineView_Previews.self,
"SwipeRightAction_Previews" : SwipeRightAction_Previews.self,
"SwipeToReplyView_Previews" : SwipeToReplyView_Previews.self,
"TextRoomTimelineView_Previews" : TextRoomTimelineView_Previews.self,
"ThreadDecorator_Previews" : ThreadDecorator_Previews.self,
"TimelineDeliveryStatusView_Previews" : TimelineDeliveryStatusView_Previews.self,
"TimelineItemBubbledStylerView_Previews" : TimelineItemBubbledStylerView_Previews.self,
"TimelineItemDebugView_Previews" : TimelineItemDebugView_Previews.self,
"TimelineItemMenu_Previews" : TimelineItemMenu_Previews.self,
"TimelineItemSendInfoLabel_Previews" : TimelineItemSendInfoLabel_Previews.self,
"TimelineItemStyler_Previews" : TimelineItemStyler_Previews.self,
"TimelineMediaPreviewDetailsView_Previews" : TimelineMediaPreviewDetailsView_Previews.self,
"TimelineMediaPreviewRedactConfirmationView_Previews" : TimelineMediaPreviewRedactConfirmationView_Previews.self,
"TimelineReactionView_Previews" : TimelineReactionView_Previews.self,
"TimelineReadReceiptsView_Previews" : TimelineReadReceiptsView_Previews.self,
"TimelineReplyView_Previews" : TimelineReplyView_Previews.self,
"TimelineStartRoomTimelineView_Previews" : TimelineStartRoomTimelineView_Previews.self,
"TimelineThreadSummaryView_Previews" : TimelineThreadSummaryView_Previews.self,
"TimelineView_Previews" : TimelineView_Previews.self,
"TombstonedAvatarImage_Previews" : TombstonedAvatarImage_Previews.self,
"TypingIndicatorView_Previews" : TypingIndicatorView_Previews.self,
"UnsupportedRoomTimelineView_Previews" : UnsupportedRoomTimelineView_Previews.self,
"UserDetailsEditScreen_Previews" : UserDetailsEditScreen_Previews.self,
"UserIndicatorModalView_Previews" : UserIndicatorModalView_Previews.self,
"UserIndicatorToastView_Previews" : UserIndicatorToastView_Previews.self,
"UserProfileCell_Previews" : UserProfileCell_Previews.self,
"UserProfileScreen_Previews" : UserProfileScreen_Previews.self,
"VerificationBadge_Previews" : VerificationBadge_Previews.self,
"VideoMediaEventsTimelineView_Previews" : VideoMediaEventsTimelineView_Previews.self,
"VideoRoomTimelineView_Previews" : VideoRoomTimelineView_Previews.self,
"VisualListItem_Previews" : VisualListItem_Previews.self,
"VoiceMessageButton_Previews" : VoiceMessageButton_Previews.self,
"VoiceMessageMediaEventsTimelineView_Previews" : VoiceMessageMediaEventsTimelineView_Previews.self,
"VoiceMessagePreviewComposer_Previews" : VoiceMessagePreviewComposer_Previews.self,
"VoiceMessageRecordingButton_Previews" : VoiceMessageRecordingButton_Previews.self,
"VoiceMessageRecordingComposer_Previews" : VoiceMessageRecordingComposer_Previews.self,
"VoiceMessageRecordingView_Previews" : VoiceMessageRecordingView_Previews.self,
"VoiceMessageRoomPlaybackView_Previews" : VoiceMessageRoomPlaybackView_Previews.self,
"VoiceMessageRoomTimelineView_Previews" : VoiceMessageRoomTimelineView_Previews.self,
"WaveformCursorView_Previews" : WaveformCursorView_Previews.self,
]
}
// swiftlint:enable all
// swiftformat:enable all
@@ -29,6 +29,16 @@ enum UITestsSignal: Codable, Equatable {
/// Posts a notification.
case notification(name: Notification.Name)
case accessibilityAudit(AccessibilityAudit)
enum AccessibilityAudit: Codable, Equatable {
/// Ask the app for the next preview.
case nextPreview
/// Tell the test runner about a loaded preview.
case nextPreviewReady(name: String)
/// Tell the test runner that there are no more previews.
case noMorePreviews
}
}
enum UITestsSignalError: String, LocalizedError {
+12
View File
@@ -180,6 +180,18 @@ targets:
else
echo "warning: Sourcery not installed, run swift run tools setup-project"
fi
- name: 🧙 Sourcery - Generate Accessibility Tests
runOnlyWhenInstalling: false
basedOnDependencyAnalysis: false
shell: /bin/sh
script: |
export PATH="$PATH:/opt/homebrew/bin"
if which sourcery >/dev/null; then
sourcery --config Tools/Sourcery/TestablePreviewsDictionary.yml
else
echo "warning: Sourcery not installed, run swift run tools setup-project"
fi
postBuildScripts:
- name: ⚠️ SwiftLint