Media uploading (#851)

* MediaUploadingPreprocessor - Prevent images without any GPS metadata from being changed

* Add support for sending images

* Add support for sending video, audio and file attachments

* Rename MediaPickerPreview to MediaUploadPreview

* Move media uploading to the MediaUploadPreviewScreen, add waiting indicators and error handling

* Add support for pasting and drag&dropping in media for upload

* Adopt new media picker source UI, remove developer flag

* Set minimum heights for timeline loadable images

* Fix invalid camera picker file names

* Fix flakey MediaUploadingPreprocessor image tests, improve gps metadata stripping

* UITests: Update existing screenshots and add new step for the room attachment picker

* Switch all github action runners to macos-13

* Cleanup enter key and paste message composer handlers
This commit is contained in:
Stefan Ceriu
2023-05-04 16:09:29 +03:00
committed by GitHub
parent e9c07c9f8f
commit 93e3a1a80b
100 changed files with 809 additions and 393 deletions
@@ -68,6 +68,8 @@ final class RoomScreenCoordinator: CoordinatorProtocol {
self.displayMediaPickerWithSource(.photoLibrary)
case .displayDocumentPicker:
self.displayMediaPickerWithSource(.documents)
case .displayMediaUploadPreviewScreen(let url):
self.displayMediaUploadPreviewScreenForFile(at: url)
}
}
}
@@ -84,44 +86,43 @@ final class RoomScreenCoordinator: CoordinatorProtocol {
// MARK: - Private
private func displayMediaPickerWithSource(_ source: MediaPickerScreenSource) {
let mediaPickerCoordinator = MediaPickerScreenCoordinator(source: source) { [weak self] action in
let stackCoordinator = NavigationStackCoordinator()
let userIndicatorController = UserIndicatorController(rootCoordinator: stackCoordinator)
let mediaPickerCoordinator = MediaPickerScreenCoordinator(userIndicatorController: userIndicatorController, source: source) { [weak self] action in
switch action {
case .cancel:
self?.navigationStackCoordinator.setSheetCoordinator(nil)
case .error:
break
case .selectMediaAtURL(let url):
let mediaPickerPreviewScreenCoordinator = MediaPickerPreviewScreenCoordinator(parameters: .init(url: url, title: url.lastPathComponent)) { action in
switch action {
case .send:
Task {
let preprocessor = MediaUploadingPreprocessor()
switch await preprocessor.processMedia(at: url) {
case .success(let mediaInfo):
MXLog.info(mediaInfo)
switch mediaInfo {
case let .image(imageURL, thumbnailURL, imageInfo):
let _ = await self?.parameters.roomProxy.sendImage(url: imageURL, thumbnailURL: thumbnailURL, imageInfo: imageInfo)
default:
break
}
case .failure(let error):
MXLog.error("Failed processing media to upload with error: \(error)")
}
self?.navigationStackCoordinator.setSheetCoordinator(nil)
}
case .cancel:
self?.navigationStackCoordinator.setSheetCoordinator(nil)
}
}
self?.navigationStackCoordinator.setSheetCoordinator(mediaPickerPreviewScreenCoordinator)
self?.displayMediaUploadPreviewScreenForFile(at: url)
}
}
navigationStackCoordinator.setSheetCoordinator(mediaPickerCoordinator)
stackCoordinator.setRootCoordinator(mediaPickerCoordinator)
navigationStackCoordinator.setSheetCoordinator(userIndicatorController)
}
private func displayMediaUploadPreviewScreenForFile(at url: URL) {
let stackCoordinator = NavigationStackCoordinator()
let userIndicatorController = UserIndicatorController(rootCoordinator: stackCoordinator)
let parameters = MediaUploadPreviewScreenCoordinatorParameters(userIndicatorController: userIndicatorController,
roomProxy: parameters.roomProxy,
mediaUploadingPreprocessor: MediaUploadingPreprocessor(),
title: url.lastPathComponent,
url: url)
let mediaUploadPreviewScreenCoordinator = MediaUploadPreviewScreenCoordinator(parameters: parameters) { [weak self] action in
switch action {
case .dismiss:
self?.navigationStackCoordinator.setSheetCoordinator(nil)
}
}
stackCoordinator.setRootCoordinator(mediaUploadPreviewScreenCoordinator)
navigationStackCoordinator.setSheetCoordinator(userIndicatorController)
}
private func displayFilePreview(for file: MediaFileHandleProxy, with title: String?) {
@@ -26,6 +26,7 @@ enum RoomScreenViewModelAction {
case displayCameraPicker
case displayMediaPicker
case displayDocumentPicker
case displayMediaUploadPreviewScreen(url: URL)
}
enum RoomScreenComposerMode: Equatable {
@@ -62,6 +63,8 @@ enum RoomScreenViewAction {
case displayCameraPicker
case displayMediaPicker
case displayDocumentPicker
case handlePasteOrDrop(provider: NSItemProvider)
}
struct RoomScreenViewState: BindableState {
@@ -73,7 +76,6 @@ struct RoomScreenViewState: BindableState {
var isBackPaginating = false
var showLoading = false
var timelineStyle: TimelineStyle
var mediaUploadingFlowEnabled: Bool
var bindings: RoomScreenViewStateBindings
@@ -39,7 +39,6 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
roomTitle: roomName ?? "Unknown room 💥",
roomAvatarURL: roomAvatarUrl,
timelineStyle: ServiceLocator.shared.settings.timelineStyle,
mediaUploadingFlowEnabled: ServiceLocator.shared.settings.mediaUploadingFlowEnabled,
bindings: .init(composerText: "", composerFocused: false)),
imageProvider: mediaProvider)
@@ -74,11 +73,7 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
ServiceLocator.shared.settings.$timelineStyle
.weakAssign(to: \.state.timelineStyle, on: self)
.store(in: &cancellables)
ServiceLocator.shared.settings.$mediaUploadingFlowEnabled
.weakAssign(to: \.state.mediaUploadingFlowEnabled, on: self)
.store(in: &cancellables)
buildTimelineViews()
}
@@ -122,6 +117,8 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
callback?(.displayMediaPicker)
case .displayDocumentPicker:
callback?(.displayDocumentPicker)
case .handlePasteOrDrop(let provider):
handlePasteOrDrop(provider)
}
}
@@ -334,6 +331,60 @@ class RoomScreenViewModel: RoomScreenViewModelType, RoomScreenViewModelProtocol
state.composerMode = .default
}
}
// Pasting and dropping
private func handlePasteOrDrop(_ provider: NSItemProvider) {
guard let type = provider.registeredContentTypes.first,
let preferredExtension = type.preferredFilenameExtension else {
MXLog.error("Invalid NSItemProvider: \(provider)")
displayError(.toast(L10n.screenRoomErrorFailedProcessingMedia))
return
}
let providerSuggestedName = provider.suggestedName
let providerDescription = provider.description
_ = provider.loadDataRepresentation(for: type) { data, error in
Task { @MainActor in
let loadingIndicatorIdentifier = UUID().uuidString
ServiceLocator.shared.userIndicatorController.submitIndicator(UserIndicator(id: loadingIndicatorIdentifier, type: .modal, title: L10n.commonLoading, persistent: true))
defer {
ServiceLocator.shared.userIndicatorController.retractIndicatorWithId(loadingIndicatorIdentifier)
}
if let error {
self.displayError(.toast(L10n.screenRoomErrorFailedProcessingMedia))
MXLog.error("Failed processing NSItemProvider: \(providerDescription) with error: \(error)")
return
}
guard let data else {
self.displayError(.toast(L10n.screenRoomErrorFailedProcessingMedia))
MXLog.error("Invalid NSItemProvider data: \(providerDescription)")
return
}
do {
let url = try await Task.detached {
if let filename = providerSuggestedName {
let hasExtension = !(filename as NSString).pathExtension.isEmpty
let filename = hasExtension ? filename : "\(filename).\(preferredExtension)"
return try FileManager.default.writeDataToTemporaryDirectory(data: data, fileName: filename)
} else {
let filename = "\(UUID().uuidString).\(preferredExtension)"
return try FileManager.default.writeDataToTemporaryDirectory(data: data, fileName: filename)
}
}.value
self.callback?(.displayMediaUploadPreviewScreen(url: url))
} catch {
self.displayError(.toast(L10n.screenRoomErrorFailedProcessingMedia))
MXLog.error("Failed storing NSItemProvider data \(providerDescription) with error: \(error)")
}
}
}
}
}
// MARK: - Mocks
@@ -22,7 +22,8 @@ struct MessageComposer: View {
let sendingDisabled: Bool
let type: RoomScreenComposerMode
let sendAction: () -> Void
let sendAction: EnterKeyHandler
let pasteAction: PasteHandler
let replyCancellationAction: () -> Void
let editCancellationAction: () -> Void
@@ -39,7 +40,8 @@ struct MessageComposer: View {
focused: $focused,
isMultiline: $isMultiline,
maxHeight: 300,
onEnterKeyHandler: sendAction)
enterKeyHandler: sendAction,
pasteHandler: pasteAction)
.tint(.element.brand)
.padding(.vertical, 10)
@@ -173,6 +175,7 @@ struct MessageComposer_Previews: PreviewProvider {
sendingDisabled: true,
type: .default,
sendAction: { },
pasteAction: { _ in },
replyCancellationAction: { },
editCancellationAction: { })
@@ -181,6 +184,7 @@ struct MessageComposer_Previews: PreviewProvider {
sendingDisabled: false,
type: .default,
sendAction: { },
pasteAction: { _ in },
replyCancellationAction: { },
editCancellationAction: { })
@@ -189,6 +193,7 @@ struct MessageComposer_Previews: PreviewProvider {
sendingDisabled: false,
type: .default,
sendAction: { },
pasteAction: { _ in },
replyCancellationAction: { },
editCancellationAction: { })
@@ -197,6 +202,7 @@ struct MessageComposer_Previews: PreviewProvider {
sendingDisabled: false,
type: .default,
sendAction: { },
pasteAction: { _ in },
replyCancellationAction: { },
editCancellationAction: { })
@@ -206,6 +212,7 @@ struct MessageComposer_Previews: PreviewProvider {
type: .reply(id: UUID().uuidString,
displayName: "John Doe"),
sendAction: { },
pasteAction: { _ in },
replyCancellationAction: { },
editCancellationAction: { })
@@ -214,6 +221,7 @@ struct MessageComposer_Previews: PreviewProvider {
sendingDisabled: false,
type: .edit(originalItemId: UUID().uuidString),
sendAction: { },
pasteAction: { _ in },
replyCancellationAction: { },
editCancellationAction: { })
}
@@ -16,7 +16,8 @@
import SwiftUI
typealias OnEnterKeyHandler = () -> Void
typealias EnterKeyHandler = () -> Void
typealias PasteHandler = (NSItemProvider) -> Void
struct MessageComposerTextField: View {
let placeholder: String
@@ -25,7 +26,8 @@ struct MessageComposerTextField: View {
@Binding var isMultiline: Bool
let maxHeight: CGFloat
let onEnterKeyHandler: OnEnterKeyHandler
let enterKeyHandler: EnterKeyHandler
let pasteHandler: PasteHandler
private var showingPlaceholder: Bool {
text.isEmpty
@@ -40,7 +42,8 @@ struct MessageComposerTextField: View {
focused: $focused,
isMultiline: $isMultiline,
maxHeight: maxHeight,
onEnterKeyHandler: onEnterKeyHandler)
enterKeyHandler: enterKeyHandler,
pasteHandler: pasteHandler)
.accessibilityLabel(placeholder)
.background(placeholderView, alignment: .topLeading)
}
@@ -64,15 +67,16 @@ private struct UITextViewWrapper: UIViewRepresentable {
let maxHeight: CGFloat
let onEnterKeyHandler: OnEnterKeyHandler
let enterKeyHandler: EnterKeyHandler
let pasteHandler: PasteHandler
private let font = UIFont.preferredFont(forTextStyle: .body)
func makeUIView(context: UIViewRepresentableContext<UITextViewWrapper>) -> UITextView {
let textView = TextViewWithKeyDetection()
let textView = ElementTextView()
textView.isMultiline = $isMultiline
textView.delegate = context.coordinator
textView.keyDelegate = context.coordinator
textView.elementDelegate = context.coordinator
textView.textColor = .element.primaryContent
textView.isEditable = true
textView.font = font
@@ -126,25 +130,29 @@ private struct UITextViewWrapper: UIViewRepresentable {
Coordinator(text: $text,
focused: $focused,
maxHeight: maxHeight,
onEnterKeyHandler: onEnterKeyHandler)
enterKeyHandler: enterKeyHandler,
pasteHandler: pasteHandler)
}
final class Coordinator: NSObject, UITextViewDelegate, TextViewWithKeyDetectionDelegate {
final class Coordinator: NSObject, UITextViewDelegate, ElementTextViewDelegate {
private var text: Binding<String>
private var focused: Binding<Bool>
private let maxHeight: CGFloat
private let onEnterKeyHandler: OnEnterKeyHandler
private let enterKeyHandler: EnterKeyHandler
private let pasteHandler: PasteHandler
init(text: Binding<String>,
focused: Binding<Bool>,
maxHeight: CGFloat,
onEnterKeyHandler: @escaping OnEnterKeyHandler) {
enterKeyHandler: @escaping EnterKeyHandler,
pasteHandler: @escaping PasteHandler) {
self.text = text
self.focused = focused
self.maxHeight = maxHeight
self.onEnterKeyHandler = onEnterKeyHandler
self.enterKeyHandler = enterKeyHandler
self.pasteHandler = pasteHandler
}
func textViewDidChange(_ textView: UITextView) {
@@ -159,23 +167,28 @@ private struct UITextViewWrapper: UIViewRepresentable {
focused.wrappedValue = false
}
func enterKeyWasPressed(textView: UITextView) {
onEnterKeyHandler()
func textViewDidReceiveEnterKeyPress(_ textView: UITextView) {
enterKeyHandler()
}
func shiftEnterKeyPressed(textView: UITextView) {
func textViewDidReceiveShiftEnterKeyPress(_ textView: UITextView) {
textView.insertText("\n")
}
func textView(_ textView: UITextView, didReceivePasteWith provider: NSItemProvider) {
pasteHandler(provider)
}
}
}
private protocol TextViewWithKeyDetectionDelegate: AnyObject {
func enterKeyWasPressed(textView: UITextView)
func shiftEnterKeyPressed(textView: UITextView)
private protocol ElementTextViewDelegate: AnyObject {
func textViewDidReceiveShiftEnterKeyPress(_ textView: UITextView)
func textViewDidReceiveEnterKeyPress(_ textView: UITextView)
func textView(_ textView: UITextView, didReceivePasteWith provider: NSItemProvider)
}
private class TextViewWithKeyDetection: UITextView {
weak var keyDelegate: TextViewWithKeyDetectionDelegate?
private class ElementTextView: UITextView {
weak var elementDelegate: ElementTextViewDelegate?
var isMultiline: Binding<Bool>?
@@ -185,11 +198,11 @@ private class TextViewWithKeyDetection: UITextView {
}
@objc func shiftEnterKeyPressed(sender: UIKeyCommand) {
keyDelegate?.shiftEnterKeyPressed(textView: self)
elementDelegate?.textViewDidReceiveShiftEnterKeyPress(self)
}
@objc func enterKeyPressed(sender: UIKeyCommand) {
keyDelegate?.enterKeyWasPressed(textView: self)
elementDelegate?.textViewDidReceiveEnterKeyPress(self)
}
override func layoutSubviews() {
@@ -208,6 +221,31 @@ private class TextViewWithKeyDetection: UITextView {
}
}
}
// Pasting support
override func canPerformAction(_ action: Selector, withSender sender: Any?) -> Bool {
if super.canPerformAction(action, withSender: sender) {
return true
}
guard action == #selector(paste(_:)) else {
return false
}
return UIPasteboard.general.itemProviders.first?.isSupportedForPasteOrDrop ?? false
}
override func paste(_ sender: Any?) {
super.paste(sender)
guard let provider = UIPasteboard.general.itemProviders.first,
provider.isSupportedForPasteOrDrop else {
return
}
elementDelegate?.textView(self, didReceivePasteWith: provider)
}
}
struct MessageComposerTextField_Previews: PreviewProvider {
@@ -236,7 +274,8 @@ struct MessageComposerTextField_Previews: PreviewProvider {
focused: $focused,
isMultiline: $isMultiline,
maxHeight: 300,
onEnterKeyHandler: { })
enterKeyHandler: { },
pasteHandler: { _ in })
}
}
}
@@ -0,0 +1,98 @@
//
// Copyright 2023 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
import SwiftUI
struct RoomAttachmentPicker: View {
@ObservedObject var context: RoomScreenViewModel.Context
@State private var showAttachmentPopover = false
@State private var sheetContentHeight = CGFloat(0)
var body: some View {
Button {
showAttachmentPopover = true
} label: {
Image(systemName: "plus.circle.fill")
.font(.compound.headingLG)
.foregroundColor(.element.accent)
}
.accessibilityIdentifier(A11yIdentifiers.roomScreen.attachmentPicker)
.popover(isPresented: $showAttachmentPopover) {
VStack(alignment: .leading, spacing: 0.0) {
Button {
showAttachmentPopover = false
context.send(viewAction: .displayMediaPicker)
} label: {
PickerLabel(title: L10n.screenRoomAttachmentSourceGallery, systemImageName: "photo.fill")
}
Button {
showAttachmentPopover = false
context.send(viewAction: .displayDocumentPicker)
} label: {
PickerLabel(title: L10n.screenRoomAttachmentSourceFiles, systemImageName: "paperclip")
}
Button {
showAttachmentPopover = false
context.send(viewAction: .displayCameraPicker)
} label: {
PickerLabel(title: L10n.screenRoomAttachmentSourceCamera, systemImageName: "camera.fill")
}
}
.background {
// This is done in the background otherwise GeometryReader tends to expand to
// all the space given to it like color or shape.
GeometryReader { proxy in
Color.clear
.onAppear {
sheetContentHeight = proxy.size.height
}
}
}
.presentationDetents([.height(sheetContentHeight)])
.presentationDragIndicator(.visible)
.tint(.element.accent)
}
}
private struct PickerLabel: View {
let title: String
let systemImageName: String
var body: some View {
Label(title, systemImage: systemImageName)
.labelStyle(EqualIconWidthLabelStyle())
.multilineTextAlignment(.leading)
.frame(maxWidth: .infinity, alignment: .leading)
.padding()
}
}
}
private struct EqualIconWidthLabelStyle: LabelStyle {
@ScaledMetric private var menuIconSize = 24.0
func makeBody(configuration: Configuration) -> some View {
HStack {
configuration
.icon
.frame(width: menuIconSize, height: menuIconSize)
configuration.title
}
}
}
@@ -19,16 +19,14 @@ import SwiftUI
struct RoomScreen: View {
@ObservedObject var context: RoomScreenViewModel.Context
@State private var showReactionsMenuForItemId = ""
@State private var dragOver = false
var body: some View {
timeline
.background(Color.element.background.ignoresSafeArea()) // Kills the toolbar translucency.
.safeAreaInset(edge: .bottom, spacing: 0) {
HStack(spacing: 4.0) {
if context.viewState.mediaUploadingFlowEnabled {
sendAttachmentButton
}
RoomAttachmentPicker(context: context)
messageComposer
}
.padding()
@@ -48,6 +46,15 @@ struct RoomScreen: View {
guard !Task.isCancelled else { return }
context.send(viewAction: .markRoomAsRead)
}
.onDrop(of: ["public.item"], isTargeted: $dragOver) { providers -> Bool in
guard let provider = providers.first,
provider.isSupportedForPasteOrDrop else {
return false
}
context.send(viewAction: .handlePasteOrDrop(provider: provider))
return true
}
}
private var timeline: some View {
@@ -64,6 +71,8 @@ struct RoomScreen: View {
sendingDisabled: context.viewState.sendButtonDisabled,
type: context.viewState.composerMode) {
sendMessage()
} pasteAction: { provider in
context.send(viewAction: .handlePasteOrDrop(provider: provider))
} replyCancellationAction: {
context.send(viewAction: .cancelReply)
} editCancellationAction: {
@@ -113,30 +122,6 @@ struct RoomScreen: View {
}
}
private var sendAttachmentButton: some View {
Menu {
Button {
context.send(viewAction: .displayDocumentPicker)
} label: {
Label(UntranslatedL10n.mediaUploadDocumentPicker, systemImage: "doc")
}
Button {
context.send(viewAction: .displayMediaPicker)
} label: {
Label(UntranslatedL10n.mediaUploadPhotoAndVideoPicker, systemImage: "photo")
}
Button {
context.send(viewAction: .displayCameraPicker)
} label: {
Label(UntranslatedL10n.mediaUploadCameraPicker, systemImage: "camera")
}
} label: {
Image(systemName: "plus.circle")
.font(.compound.headingLG)
.foregroundColor(.element.brand)
}
}
private func sendMessage() {
guard !context.viewState.sendButtonDisabled else { return }
context.send(viewAction: .sendMessage)
@@ -28,7 +28,7 @@ struct ImageRoomTimelineView: View {
imageProvider: context.imageProvider) {
placeholder
}
.frame(maxHeight: min(300, timelineItem.height ?? .infinity))
.frame(maxHeight: min(300, max(100, timelineItem.height ?? .infinity)))
.aspectRatio(timelineItem.aspectRatio, contentMode: .fit)
}
}
@@ -28,7 +28,7 @@ struct StickerRoomTimelineView: View {
imageProvider: context.imageProvider) {
placeholder
}
.frame(maxHeight: min(300, timelineItem.height ?? .infinity))
.frame(maxHeight: min(300, max(100, timelineItem.height ?? .infinity)))
.aspectRatio(timelineItem.aspectRatio, contentMode: .fit)
}
.accessibilityLabel(timelineItem.body)
@@ -24,7 +24,7 @@ struct VideoRoomTimelineView: View {
var body: some View {
TimelineStyler(timelineItem: timelineItem) {
thumbnail
.frame(maxHeight: min(300, timelineItem.height ?? .infinity))
.frame(maxHeight: min(300, max(100, timelineItem.height ?? .infinity)))
.aspectRatio(timelineItem.aspectRatio, contentMode: .fit)
}
}