Content scanner (Element Pro): scan timeline events for unsafe content (#7159)

* Simplify how event content with captions are displayed

The caption component will now be added in `TimelineItemEventContentView` instead of separately in each `TimelineItem__View` component

* Make sure `CoilMediaFetcher` methods to fetch media throw errors

We need to catch them later to check if the content scanner marked some thumbnails or medias as invalid/unsafe in `AsyncImage` instances

* Add a `ContentScannerService` that allows us to manually scan a list of medias in an event and then update and store the results

The results take the form of `ContentValidationValue`, stored in a `ContentValidationState` instance tied to the event

* Create `EventContentValidationCache` to keep the `ContentValidationStates` for events in a room

* Expose `EventContentValidationCache` to Compose using `LocalEventContentValidationState`

Also add a couple of helper functions to avoid having to manually remember the states

* Use the new components to display invalid content in the timeline items

* Add `blurhash` to most media items so we can use it while validating the thumbnails/medias

* Fix custom layout issues with screenshot and UI tests

* Also check the validation state for replied-to events in the composer

* Fix and add tests

* Handle unrecoverable content validation errors too: display a 'not found' UI

* Improve the performance of `EqualWidthColumn` for the single item case

* Update screenshots

---------

Co-authored-by: ElementBot <android@element.io>
This commit is contained in:
Jorge Martin Espinosa
2026-07-16 09:59:51 +02:00
committed by GitHub
parent 56cf00560e
commit 0d03472c60
241 changed files with 2881 additions and 828 deletions
@@ -0,0 +1,93 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.ui.media.contentvalidation
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.BoxWithConstraints
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.material3.Icon
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.movableContentOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberUpdatedState
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.text.TextLayoutResult
import androidx.compose.ui.text.rememberTextMeasurer
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import io.element.android.compound.theme.ElementTheme
import io.element.android.compound.tokens.generated.CompoundIcons
@Composable
internal fun ContentErrorView(
title: String,
message: String,
modifier: Modifier = Modifier,
contentPadding: PaddingValues = PaddingValues.Zero,
onTextLayout: ((TextLayoutResult) -> Unit)? = null,
) {
val updatedOnTextLayout by rememberUpdatedState(onTextLayout)
Row(
modifier = modifier
.background(color = ElementTheme.colors.bgCriticalSubtle)
.padding(contentPadding),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.CenterVertically
) {
Icon(
modifier = Modifier.size(24.dp),
imageVector = CompoundIcons.Error(),
contentDescription = null,
tint = ElementTheme.colors.iconCriticalPrimary
)
Column(Modifier.weight(1f)) {
Text(
text = title,
color = ElementTheme.colors.textCriticalPrimary,
style = ElementTheme.typography.fontBodyMdMedium,
)
val textMeasurer = rememberTextMeasurer()
val textContent = remember(message) {
movableContentOf(
@Composable {
Text(
text = message,
color = ElementTheme.colors.textSecondary,
style = ElementTheme.typography.fontBodySmRegular,
)
}
)
}
// BoxWithConstraints is needed to be able to calculate the text layout in the current constraints, so we can pass it to the onTextLayout callback.
// However, this can't be used inside a SubComposeLayout (like the one in the text composer), so we only use it when the onTextLayout callback
// is provided.
if (updatedOnTextLayout != null) {
BoxWithConstraints(modifier = Modifier.fillMaxWidth()) {
LaunchedEffect(message) {
updatedOnTextLayout?.invoke(textMeasurer.measure(message, overflow = TextOverflow.Visible, constraints = constraints))
}
textContent()
}
} else {
textContent()
}
}
}
}
@@ -0,0 +1,197 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.ui.media.contentvalidation
import androidx.compose.runtime.Composable
import androidx.compose.runtime.Immutable
import androidx.compose.runtime.State
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.produceState
import androidx.compose.runtime.remember
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.mapLatest
import kotlinx.coroutines.flow.update
/**
* Wrapper for the content validation state of an event.
*
* It can be used to check the overall validation state of the event, as well as the validation state of each media item in it.
*/
@Immutable
interface ContentValidationState {
/**
* A flow that emits the overall validation state of the event, which is an aggregation of the different media values.
*/
val overallStateFlow: Flow<ContentValidationValue>
/**
* Returns a flow that emits the validation state of a specific media item, identified by its URL.
*/
fun getMediaStateFlow(url: String): Flow<ContentValidationValue>
/**
* Returns the current overall validation state of the event.
*/
fun getCurrentOverallState(): ContentValidationValue
/**
* Returns the current validation state of a specific media item, identified by its URL.
*/
fun getCurrentMediaState(url: String): ContentValidationValue
/**
* Updates the validation state of a specific media item, identified by its URL, with a new [ContentValidationValue].
*/
fun update(url: String, newValue: ContentValidationValue)
}
/**
* A no-op implementation of [ContentValidationState] that always returns a fixed validation state.
* By default, this is [ContentValidationValue.Valid].
*/
@Immutable
class NoopContentValidationState(
private val initial: ContentValidationValue = ContentValidationValue.Valid,
) : ContentValidationState {
override val overallStateFlow: Flow<ContentValidationValue> = MutableStateFlow(initial)
override fun getMediaStateFlow(url: String): Flow<ContentValidationValue> = MutableStateFlow(initial)
override fun getCurrentOverallState(): ContentValidationValue = initial
override fun getCurrentMediaState(url: String): ContentValidationValue = initial
override fun update(url: String, newValue: ContentValidationValue) {}
}
/**
* A default implementation of [ContentValidationState] that tracks the validation state of media items in a mutable map.
* It calculates the overall validation state based on the individual media states.
*/
@Immutable
class DefaultContentValidationState(
private val states: MutableStateFlow<Map<String, ContentValidationValue>>,
) : ContentValidationState {
@OptIn(ExperimentalCoroutinesApi::class)
override val overallStateFlow: Flow<ContentValidationValue> = states
.mapLatest { states -> calculateOverallState(states) }
.distinctUntilChanged()
constructor() : this(states = MutableStateFlow(emptyMap()))
constructor(initial: Map<String, ContentValidationValue>) : this(states = MutableStateFlow(initial))
@OptIn(ExperimentalCoroutinesApi::class)
override fun getMediaStateFlow(url: String): Flow<ContentValidationValue> {
return states.map { states ->
states[url] ?: ContentValidationValue.Unknown
}.distinctUntilChanged()
}
override fun getCurrentOverallState(): ContentValidationValue {
return calculateOverallState(states.value)
}
override fun getCurrentMediaState(url: String): ContentValidationValue {
return states.value[url] ?: ContentValidationValue.Unknown
}
override fun update(url: String, newValue: ContentValidationValue) {
states.update { current ->
if (current[url]?.isValidated() == true && newValue.isLoading()) {
// If the current state is already validated and the new value is loading, we keep the current validated state
current
} else {
// Otherwise, we update the state for the given URL with the new value
current + (url to newValue)
}
}
}
}
private fun calculateOverallState(states: Map<String, ContentValidationValue>): ContentValidationValue {
return when {
states.values.any { it is ContentValidationValue.Invalid } -> ContentValidationValue.Invalid
states.values.any { it is ContentValidationValue.Loading } -> ContentValidationValue.Loading
states.values.any { it is ContentValidationValue.UnrecoverableError } -> states.values.first { it is ContentValidationValue.UnrecoverableError }
states.values.all { it is ContentValidationValue.Valid } -> ContentValidationValue.Valid
else -> ContentValidationValue.Unknown
}
}
/**
* Represents the validation state of a media item or an event.
*/
@Immutable
sealed interface ContentValidationValue {
/** Indicates that the content is valid. */
data object Valid : ContentValidationValue
/** Indicates that the content is invalid/unsafe. */
data object Invalid : ContentValidationValue
/** Indicates that the content is currently being validated. */
data object Loading : ContentValidationValue
/** Indicates that an unrecoverable error occurred during validation, and it should not be retried. */
data class UnrecoverableError(val error: Throwable) : ContentValidationValue
/** Indicates that the validation state is unknown and should be checked. Recoverable errors are also represented with this. */
data object Unknown : ContentValidationValue
/**
* Returns true if the validation state is either [Valid], [Invalid], or [UnrecoverableError], meaning that the validation process has completed
* and a definitive result is available.
*/
fun isValidated(): Boolean = when (this) {
is Valid -> true
is Invalid -> true
is Loading -> false
is UnrecoverableError -> true
is Unknown -> false
}
/** Returns true if the validation state is [Valid]. */
fun isValid(): Boolean = this is Valid
/** Returns true if the validation state is [Invalid]. */
fun isInvalid(): Boolean = this is Invalid
/** Returns true if the validation state is [Loading]. */
fun isLoading(): Boolean = this is Loading
/** Returns true if the validation state is [UnrecoverableError]. */
fun hasUnrecoverableError(): Boolean = this is UnrecoverableError
/** Returns true if the validation state is either [Invalid] or [UnrecoverableError]. */
fun hasError(): Boolean = this is Invalid || this is UnrecoverableError
}
/**
* Collects the overall validation state of the [ContentValidationState] as a [State] in a Composable context.
*/
@Composable
fun ContentValidationState.collectOverallState(): State<ContentValidationValue> {
return produceState(getCurrentOverallState()) {
overallStateFlow.collect { value = it }
}
}
/**
* Collects the validation state of a specific media item, identified by its URL, as a [State] in a Composable context.
*/
@Composable
fun ContentValidationState.collectMediaState(url: String?): State<ContentValidationValue> {
if (url == null) {
return remember { mutableStateOf(ContentValidationValue.Unknown) }
}
return produceState(getCurrentMediaState(url), key1 = url) {
getMediaStateFlow(url).collect { value = it }
}
}
@@ -0,0 +1,20 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.ui.media.contentvalidation
import io.element.android.libraries.matrix.api.core.EventId
/**
* A cache that holds the validation state of event contents, based on their [EventId].
*/
interface EventContentValidationCache {
/**
* Returns the [ContentValidationState] for the given [eventId]. If none exists, it creates a new one with [ContentValidationValue.Unknown].
*/
operator fun get(eventId: EventId): ContentValidationState
}
@@ -0,0 +1,29 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.ui.media.contentvalidation
import io.element.android.libraries.matrix.api.core.EventId
/**
* In-memory variant of [EventContentValidationCache] that holds the validation state of event contents, based on their [EventId].
*
* Note: this is intended to be used in tests.
*/
class InMemoryEventContentValidationCache(
initial: Map<EventId, ContentValidationState> = emptyMap(),
) : EventContentValidationCache {
private val cache = initial.toMutableMap()
override operator fun get(eventId: EventId): ContentValidationState {
return cache[eventId] ?: DefaultContentValidationState()
}
fun put(eventId: EventId, state: ContentValidationState) {
cache[eventId] = state
}
}
@@ -0,0 +1,46 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.ui.media.contentvalidation
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.TextLayoutResult
import androidx.compose.ui.unit.dp
import io.element.android.libraries.designsystem.preview.ElementPreview
import io.element.android.libraries.designsystem.preview.PreviewsDayNight
import io.element.android.libraries.ui.strings.CommonStrings
/**
* A view to display when the media content is invalid or dangerous.
*/
@Composable
fun InvalidContentView(
modifier: Modifier = Modifier,
contentPadding: PaddingValues = PaddingValues.Zero,
onTextLayout: ((TextLayoutResult) -> Unit)? = null,
) {
ContentErrorView(
title = stringResource(CommonStrings.content_scanner_unsafe_title),
message = stringResource(CommonStrings.content_scanner_unsafe_message),
modifier = modifier,
contentPadding = contentPadding,
onTextLayout = onTextLayout,
)
}
@PreviewsDayNight
@Composable
internal fun InvalidContentViewPreview() = ElementPreview {
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
InvalidContentView()
}
}
@@ -0,0 +1,72 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.ui.media.contentvalidation
import androidx.compose.runtime.Composable
import androidx.compose.runtime.remember
import androidx.compose.runtime.staticCompositionLocalOf
import io.element.android.libraries.matrix.api.core.EventId
import io.element.android.libraries.matrix.api.timeline.item.event.EventContent
import io.element.android.libraries.matrix.api.timeline.item.event.MessageContent
import io.element.android.libraries.matrix.api.timeline.item.event.StickerContent
import io.element.android.libraries.matrix.api.timeline.item.event.isMediaContent
private val noopValue = NoopContentValidationState()
/**
* A composition local that provides an [EventContentValidationCache] instance, so we can check if a given event content has been validated or not.
*/
val LocalEventContentValidationState = staticCompositionLocalOf<EventContentValidationCache> { NoopEventContentValidationCache() }
/**
* A noop implementation of the [EventContentValidationCache] that immediately returns a successful validation state. This will be used in FOSS.
*/
class NoopEventContentValidationCache : EventContentValidationCache {
override operator fun get(eventId: EventId): ContentValidationState = noopValue
}
/**
* A helper to remember the validation state of an event content, based on its [EventId] and [EventContent], if known.
*/
@Composable
fun rememberEventContentValidationState(eventId: EventId?, eventContent: EventContent?): ContentValidationState {
val needsValidation = remember(eventContent) {
when (eventContent) {
is MessageContent -> eventContent.isMediaContent()
is StickerContent -> true
// If the event content is not known or not a media content, we don't need to validate it.
else -> false
}
}
return rememberEventContentValidationState(eventId, needsValidation)
}
/**
* A helper to remember the validation state of an event content, based on its [EventId] and whether it needs validation or not.
*/
@Composable
fun rememberEventContentValidationState(eventId: EventId?, needsValidation: Boolean): ContentValidationState {
if (!needsValidation) {
return noopValue
}
return rememberEventContentValidationStateInternal(eventId)
}
@Composable
private fun rememberEventContentValidationStateInternal(eventId: EventId?): ContentValidationState {
val cache = LocalEventContentValidationState.current
return remember(eventId) {
if (eventId != null) {
cache[eventId]
} else {
noopValue
}
}
}
@@ -0,0 +1,46 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.ui.media.contentvalidation
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.TextLayoutResult
import androidx.compose.ui.unit.dp
import io.element.android.libraries.designsystem.preview.ElementPreview
import io.element.android.libraries.designsystem.preview.PreviewsDayNight
import io.element.android.libraries.ui.strings.CommonStrings
/**
* A view to display when the media content can't be fetched.
*/
@Composable
fun NotFoundContentView(
modifier: Modifier = Modifier,
contentPadding: PaddingValues = PaddingValues.Zero,
onTextLayout: ((TextLayoutResult) -> Unit)? = null,
) {
ContentErrorView(
title = stringResource(CommonStrings.content_scanner_not_found_title),
message = stringResource(CommonStrings.content_scanner_not_found),
modifier = modifier,
contentPadding = contentPadding,
onTextLayout = onTextLayout,
)
}
@PreviewsDayNight
@Composable
internal fun NotFoundContentViewPreview() = ElementPreview {
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
NotFoundContentView()
}
}
@@ -40,6 +40,12 @@ fun InReplyToDetails.eventId() = when (this) {
is InReplyToDetails.Error -> eventId
}
fun InReplyToDetails.content() = when (this) {
is InReplyToDetails.Ready -> eventContent
is InReplyToDetails.Loading -> null
is InReplyToDetails.Error -> null
}
fun InReplyTo.map(
permalinkParser: PermalinkParser,
) = when (this) {
@@ -19,6 +19,7 @@ import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
@@ -44,6 +45,9 @@ import io.element.android.libraries.matrix.api.core.UserId
import io.element.android.libraries.matrix.api.timeline.item.event.ProfileDetails
import io.element.android.libraries.matrix.api.timeline.item.event.getDisambiguatedDisplayName
import io.element.android.libraries.matrix.ui.components.AttachmentThumbnail
import io.element.android.libraries.matrix.ui.media.contentvalidation.ContentValidationValue
import io.element.android.libraries.matrix.ui.media.contentvalidation.InvalidContentView
import io.element.android.libraries.matrix.ui.media.contentvalidation.NotFoundContentView
import io.element.android.libraries.matrix.ui.messages.sender.SenderName
import io.element.android.libraries.matrix.ui.messages.sender.SenderNameMode
import io.element.android.libraries.ui.strings.CommonStrings
@@ -55,18 +59,22 @@ import io.element.android.libraries.ui.strings.CommonStrings
fun InReplyToView(
inReplyTo: InReplyToDetails,
hideImage: Boolean,
contentValidationValue: ContentValidationValue,
modifier: Modifier = Modifier,
maxLines: Int = 2,
) {
when (inReplyTo) {
is InReplyToDetails.Ready -> {
ReplyToReadyContent(
is InReplyToDetails.Ready -> when (contentValidationValue) {
ContentValidationValue.Valid -> ReplyToReadyContent(
senderId = inReplyTo.senderId,
senderProfile = inReplyTo.senderProfile,
metadata = inReplyTo.metadata(hideImage),
maxLines = maxLines,
modifier = modifier,
)
ContentValidationValue.Invalid -> ReplyToInvalidContent()
is ContentValidationValue.UnrecoverableError -> ReplyToNotFoundContent()
else -> ReplyToLoadingContent(modifier = modifier)
}
is InReplyToDetails.Error ->
ReplyToErrorContent(data = inReplyTo, maxLines = maxLines, modifier = modifier)
@@ -86,7 +94,7 @@ private fun ReplyToReadyContent(
val paddings = if (metadata is InReplyToMetadata.Thumbnail) {
PaddingValues(end = 8.dp)
} else {
PaddingValues(start = 8.dp, end = 8.dp)
PaddingValues(horizontal = 8.dp)
}
Row(
modifier
@@ -213,11 +221,50 @@ private fun ReplyToContentText(
}
}
@Composable
private fun ReplyToInvalidContent(
modifier: Modifier = Modifier
) {
InvalidContentView(
modifier = modifier,
contentPadding = PaddingValues(horizontal = 8.dp),
onTextLayout = null,
)
}
@Composable
private fun ReplyToNotFoundContent(
modifier: Modifier = Modifier
) {
NotFoundContentView(
modifier = modifier,
contentPadding = PaddingValues(horizontal = 8.dp),
onTextLayout = null,
)
}
@PreviewsDayNight
@Composable
internal fun InReplyToViewPreview(@PreviewParameter(provider = InReplyToDetailsProvider::class) inReplyTo: InReplyToDetails) = ElementPreview {
InReplyToView(
inReplyTo = inReplyTo,
hideImage = false,
contentValidationValue = ContentValidationValue.Valid,
)
}
@PreviewsDayNight
@Composable
internal fun ReplyToInvalidContentPreview() {
ElementPreview {
ReplyToInvalidContent(modifier = Modifier.padding(10.dp))
}
}
@PreviewsDayNight
@Composable
internal fun ReplyToNotFoundContentPreview() {
ElementPreview {
ReplyToNotFoundContent(modifier = Modifier.padding(10.dp))
}
}