Content scanner (Element Pro): scan timeline events for unsafe content (#7159)

* Simplify how event content with captions are displayed The caption component will now be added in `TimelineItemEventContentView` instead of separately in each `TimelineItem__View` component * Make sure `CoilMediaFetcher` methods to fetch media throw errors We need to catch them later to check if the content scanner marked some thumbnails or medias as invalid/unsafe in `AsyncImage` instances * Add a `ContentScannerService` that allows us to manually scan a list of medias in an event and then update and store the results The results take the form of `ContentValidationValue`, stored in a `ContentValidationState` instance tied to the event * Create `EventContentValidationCache` to keep the `ContentValidationStates` for events in a room * Expose `EventContentValidationCache` to Compose using `LocalEventContentValidationState` Also add a couple of helper functions to avoid having to manually remember the states * Use the new components to display invalid content in the timeline items * Add `blurhash` to most media items so we can use it while validating the thumbnails/medias * Fix custom layout issues with screenshot and UI tests * Also check the validation state for replied-to events in the composer * Fix and add tests * Handle unrecoverable content validation errors too: display a 'not found' UI * Improve the performance of `EqualWidthColumn` for the single item case * Update screenshots --------- Co-authored-by: ElementBot <android@element.io>
2026-11-02 12:07:28 +00:00 · 2026-07-16 09:59:51 +02:00
parent 56cf00560e
commit 0d03472c60
241 changed files with 2881 additions and 828 deletions
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.features.contentscanner.api
+
+import io.element.android.libraries.matrix.api.core.EventId
+import io.element.android.libraries.matrix.api.media.MediaSource
+import io.element.android.libraries.matrix.api.scanner.ContentScanner
+import io.element.android.libraries.matrix.ui.media.contentvalidation.ContentValidationState
+
+/**
+ * Service to perform security scans in the contents for a given event and media source.
+ *
+ * The default implementation (FOSS) always returns a valid state.
+ * This will only process media when a [ContentScanner] is provided when building the client in Pro.
+ */
+fun interface ContentScannerService {
+    fun scan(eventId: EventId, mediaSources: List<MediaSource>, contentValidationState: ContentValidationState)
+}
@@ -32,4 +32,5 @@ dependencies {
    testImplementation(libs.test.truth)
    testImplementation(libs.test.turbine)
    testImplementation(projects.libraries.matrix.test)
+    testImplementation(projects.tests.testutils)
 }
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.features.contentscanner.impl
+
+import io.element.android.features.contentscanner.api.ContentScannerService
+import io.element.android.libraries.core.coroutine.CoroutineDispatchers
+import io.element.android.libraries.matrix.api.core.EventId
+import io.element.android.libraries.matrix.api.media.MediaSource
+import io.element.android.libraries.matrix.api.scanner.ContentScanner
+import io.element.android.libraries.matrix.ui.media.contentvalidation.ContentValidationState
+import io.element.android.libraries.matrix.ui.media.contentvalidation.ContentValidationValue
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.launch
+import java.io.IOException
+
+/**
+ * Default implementation of [ContentScannerService] that uses a [ContentScanner] to scan media sources.
+ */
+class DefaultContentScannerService(
+    private val contentScanner: ContentScanner,
+    private val coroutineScope: CoroutineScope,
+    coroutineDispatchers: CoroutineDispatchers,
+) : ContentScannerService {
+    private val context = coroutineDispatchers.io.limitedParallelism(4)
+
+    override fun scan(eventId: EventId, mediaSources: List<MediaSource>, contentValidationState: ContentValidationState) {
+        for (mediaSource in mediaSources) {
+            val url = mediaSource.safeUrl
+            val currentState = contentValidationState.getCurrentMediaState(url)
+            if (currentState != ContentValidationValue.Unknown) continue
+            contentValidationState.update(url, ContentValidationValue.Loading)
+
+            coroutineScope.launch(context) {
+                contentScanner.scan(mediaSource)
+                    .onSuccess { isValid ->
+                        val contentValidationValue = if (isValid) ContentValidationValue.Valid else ContentValidationValue.Invalid
+                        contentValidationState.update(url, contentValidationValue)
+                    }
+                    .onFailure { exception ->
+                        if (exception is IOException) {
+                            // If it's an IO-related exception, we can retry later, so we don't store the error
+                            contentValidationState.update(url, ContentValidationValue.Unknown)
+                        } else {
+                            // For other exceptions, we cache the failure
+                            contentValidationState.update(url, ContentValidationValue.UnrecoverableError(exception))
+                        }
+                    }
+            }
+        }
+    }
+}
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.features.contentscanner.impl
+
+import dev.zacsweers.metro.ContributesBinding
+import dev.zacsweers.metro.SingleIn
+import io.element.android.libraries.di.RoomScope
+import io.element.android.libraries.matrix.api.core.EventId
+import io.element.android.libraries.matrix.api.scanner.ContentScanner
+import io.element.android.libraries.matrix.ui.media.contentvalidation.ContentValidationState
+import io.element.android.libraries.matrix.ui.media.contentvalidation.DefaultContentValidationState
+import io.element.android.libraries.matrix.ui.media.contentvalidation.EventContentValidationCache
+import io.element.android.libraries.matrix.ui.media.contentvalidation.NoopContentValidationState
+import timber.log.Timber
+import java.util.concurrent.ConcurrentHashMap
+
+@SingleIn(RoomScope::class)
+@ContributesBinding(RoomScope::class)
+class DefaultEventContentValidationCache(
+    private val contentScanner: ContentScanner?,
+) : EventContentValidationCache {
+    private val cache = ConcurrentHashMap<EventId, ContentValidationState>()
+
+    override operator fun get(eventId: EventId): ContentValidationState {
+        return cache.getOrPut(eventId) {
+            if (contentScanner != null) {
+                DefaultContentValidationState()
+            } else {
+                Timber.v("Content scanner is not available, returning NoopContentValidationState for eventId: $eventId")
+                NoopContentValidationState()
+            }
+        }
+    }
+}
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.features.contentscanner.impl.di
+
+import io.element.android.features.contentscanner.api.ContentScannerService
+import io.element.android.libraries.matrix.api.core.EventId
+import io.element.android.libraries.matrix.api.media.MediaSource
+import io.element.android.libraries.matrix.ui.media.contentvalidation.ContentValidationState
+import io.element.android.libraries.matrix.ui.media.contentvalidation.ContentValidationValue
+
+/**
+ * Noop implementation of [ContentScannerService] that always returns success.
+ *
+ * This is used when the content scanner feature is not enabled or available.
+ */
+class AlwaysValidContentScannerService : ContentScannerService {
+    override fun scan(eventId: EventId, mediaSources: List<MediaSource>, contentValidationState: ContentValidationState) {
+        // Always return success for the noop implementation
+        for (mediaSource in mediaSources) {
+            val url = mediaSource.safeUrl
+            contentValidationState.update(url, ContentValidationValue.Valid)
+        }
+    }
+}
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.features.contentscanner.impl.di
+
+import dev.zacsweers.metro.BindingContainer
+import dev.zacsweers.metro.ContributesTo
+import dev.zacsweers.metro.Provides
+import dev.zacsweers.metro.SingleIn
+import io.element.android.features.contentscanner.api.ContentScannerService
+import io.element.android.features.contentscanner.impl.DefaultContentScannerService
+import io.element.android.libraries.core.coroutine.CoroutineDispatchers
+import io.element.android.libraries.di.RoomScope
+import io.element.android.libraries.di.annotations.RoomCoroutineScope
+import io.element.android.libraries.matrix.api.scanner.ContentScanner
+import kotlinx.coroutines.CoroutineScope
+
+@BindingContainer
+@ContributesTo(RoomScope::class)
+object ContentScannerModule {
+    @Provides
+    @SingleIn(RoomScope::class)
+    fun providesContentScannerService(
+        contentScanner: ContentScanner?,
+        @RoomCoroutineScope coroutineScope: CoroutineScope,
+        coroutineDispatchers: CoroutineDispatchers,
+    ): ContentScannerService {
+        return if (contentScanner != null) {
+            DefaultContentScannerService(
+                contentScanner = contentScanner,
+                coroutineScope = coroutineScope,
+                coroutineDispatchers = coroutineDispatchers,
+            )
+        } else {
+            AlwaysValidContentScannerService()
+        }
+    }
+}
@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.features.contentscanner.impl
+
+import com.google.common.truth.Truth.assertThat
+import io.element.android.libraries.core.coroutine.CoroutineDispatchers
+import io.element.android.libraries.matrix.api.media.MediaSource
+import io.element.android.libraries.matrix.api.scanner.ContentScanner
+import io.element.android.libraries.matrix.test.AN_EVENT_ID
+import io.element.android.libraries.matrix.test.media.aMediaSource
+import io.element.android.libraries.matrix.test.scanner.FakeContentScanner
+import io.element.android.libraries.matrix.ui.media.contentvalidation.ContentValidationValue
+import io.element.android.libraries.matrix.ui.media.contentvalidation.DefaultContentValidationState
+import io.element.android.libraries.matrix.ui.media.contentvalidation.NoopContentValidationState
+import io.element.android.tests.testutils.lambda.lambdaRecorder
+import io.element.android.tests.testutils.lambda.value
+import io.element.android.tests.testutils.testCoroutineDispatchers
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.test.TestScope
+import kotlinx.coroutines.test.runCurrent
+import kotlinx.coroutines.test.runTest
+import org.junit.Test
+import java.io.FileNotFoundException
+
+@OptIn(ExperimentalCoroutinesApi::class)
+class DefaultContentScannerServiceTest {
+    @Test
+    fun `scan with no media sources returns valid`() = runTest {
+        val scanLambda = lambdaRecorder<MediaSource, Result<Boolean>> { Result.success(true) }
+        val scanner = FakeContentScanner(scan = scanLambda)
+        val contentValidationState = DefaultContentValidationState()
+        createDefaultContentScannerService(
+            contentScanner = scanner,
+        ).scan(
+            eventId = AN_EVENT_ID,
+            mediaSources = emptyList(),
+            contentValidationState = contentValidationState,
+        )
+
+        scanLambda.assertions().isNeverCalled()
+        assertThat(contentValidationState.getCurrentOverallState()).isEqualTo(ContentValidationValue.Valid)
+    }
+
+    @Test
+    fun `scan with single media source`() = runTest {
+        val scanLambda = lambdaRecorder<MediaSource, Result<Boolean>> { Result.success(false) }
+        val scanner = FakeContentScanner(scan = scanLambda)
+        val contentValidationState = DefaultContentValidationState()
+
+        val mediaSource = aMediaSource("https://example.com/media.jpg")
+        createDefaultContentScannerService(
+            contentScanner = scanner,
+        ).scan(
+            eventId = AN_EVENT_ID,
+            mediaSources = listOf(mediaSource),
+            contentValidationState = contentValidationState,
+        )
+
+        runCurrent()
+
+        scanLambda.assertions().isCalledOnce().with(value(mediaSource))
+        assertThat(contentValidationState.getCurrentOverallState()).isEqualTo(ContentValidationValue.Invalid)
+    }
+
+    @Test
+    fun `scan with several media sources`() = runTest {
+        val mediaSourceA = aMediaSource("https://example.com/mediaA.jpg")
+        val mediaSourceB = aMediaSource("https://example.com/mediaB.jpg")
+
+        val scanLambda = lambdaRecorder<MediaSource, Result<Boolean>> { source ->
+            when (source) {
+                mediaSourceA -> Result.success(true)
+                mediaSourceB -> Result.success(false)
+                else -> error("Unexpected media source: $source")
+            }
+        }
+        val scanner = FakeContentScanner(scan = scanLambda)
+        val contentValidationState = DefaultContentValidationState()
+
+        createDefaultContentScannerService(
+            contentScanner = scanner,
+        ).scan(
+            eventId = AN_EVENT_ID,
+            mediaSources = listOf(mediaSourceA, mediaSourceB),
+            contentValidationState = contentValidationState,
+        )
+
+        runCurrent()
+
+        scanLambda.assertions().isCalledExactly(2)
+        assertThat(contentValidationState.getMediaStateFlow(mediaSourceA.safeUrl).first()).isEqualTo(ContentValidationValue.Valid)
+        assertThat(contentValidationState.getMediaStateFlow(mediaSourceB.safeUrl).first()).isEqualTo(ContentValidationValue.Invalid)
+        assertThat(contentValidationState.overallStateFlow.first()).isEqualTo(ContentValidationValue.Invalid)
+    }
+
+    @Test
+    fun `scan won't be repeated for sources which are already being scanned`() = runTest {
+        val scanLambda = lambdaRecorder<MediaSource, Result<Boolean>> { Result.success(false) }
+        val scanner = FakeContentScanner(scan = scanLambda)
+        val contentValidationState = DefaultContentValidationState()
+
+        val mediaSource = aMediaSource("https://example.com/media.jpg")
+        val service = createDefaultContentScannerService(
+            contentScanner = scanner,
+        )
+
+        // First scan attempt
+        service.scan(
+            eventId = AN_EVENT_ID,
+            mediaSources = listOf(mediaSource),
+            contentValidationState = contentValidationState,
+        )
+
+        // Immediately after, a new scan attempt is made for the same media source
+        service.scan(
+            eventId = AN_EVENT_ID,
+            mediaSources = listOf(mediaSource),
+            contentValidationState = contentValidationState,
+        )
+
+        runCurrent()
+
+        // The scan action was done just once, and the overall state is invalid as expected
+        scanLambda.assertions().isCalledOnce().with(value(mediaSource))
+        assertThat(contentValidationState.getCurrentOverallState()).isEqualTo(ContentValidationValue.Invalid)
+    }
+
+    @Test
+    fun `scan won't be repeated for sources which already have a cached value`() = runTest {
+        val scanLambda = lambdaRecorder<MediaSource, Result<Boolean>> { Result.success(false) }
+        val scanner = FakeContentScanner(scan = scanLambda)
+        val contentValidationState = NoopContentValidationState(ContentValidationValue.Valid)
+
+        val mediaSource = aMediaSource("https://example.com/media.jpg")
+        val service = createDefaultContentScannerService(
+            contentScanner = scanner,
+        )
+
+        service.scan(
+            eventId = AN_EVENT_ID,
+            mediaSources = listOf(mediaSource),
+            contentValidationState = contentValidationState,
+        )
+
+        runCurrent()
+
+        scanLambda.assertions().isNeverCalled()
+    }
+
+    @Test
+    fun `if a scan fails with a recoverable error, the content validation value will be back to Unknown`() = runTest {
+        val scanLambda = lambdaRecorder<MediaSource, Result<Boolean>> { Result.failure(FileNotFoundException("Some IO exception")) }
+        val scanner = FakeContentScanner(scan = scanLambda)
+        val contentValidationState = DefaultContentValidationState()
+
+        val mediaSource = aMediaSource("https://example.com/media.jpg")
+        val service = createDefaultContentScannerService(
+            contentScanner = scanner,
+        )
+
+        service.scan(
+            eventId = AN_EVENT_ID,
+            mediaSources = listOf(mediaSource),
+            contentValidationState = contentValidationState,
+        )
+
+        runCurrent()
+
+        scanLambda.assertions().isCalledOnce().with(value(mediaSource))
+        assertThat(contentValidationState.getCurrentMediaState(mediaSource.safeUrl)).isEqualTo(ContentValidationValue.Unknown)
+        assertThat(contentValidationState.getCurrentOverallState()).isEqualTo(ContentValidationValue.Unknown)
+    }
+
+    @Test
+    fun `if a scan fails with an unrecoverable error, the content validation value will be back to UnrecoverableError`() = runTest {
+        val error = OutOfMemoryError("BOOM")
+        val scanLambda = lambdaRecorder<MediaSource, Result<Boolean>> { Result.failure(error) }
+        val scanner = FakeContentScanner(scan = scanLambda)
+        val contentValidationState = DefaultContentValidationState()
+
+        val mediaSource = aMediaSource("https://example.com/media.jpg")
+        val service = createDefaultContentScannerService(
+            contentScanner = scanner,
+        )
+
+        service.scan(
+            eventId = AN_EVENT_ID,
+            mediaSources = listOf(mediaSource),
+            contentValidationState = contentValidationState,
+        )
+
+        runCurrent()
+
+        scanLambda.assertions().isCalledOnce().with(value(mediaSource))
+        assertThat(contentValidationState.getCurrentMediaState(mediaSource.safeUrl)).isEqualTo(ContentValidationValue.UnrecoverableError(error))
+        assertThat(contentValidationState.getCurrentOverallState()).isEqualTo(ContentValidationValue.UnrecoverableError(error))
+    }
+
+    private fun TestScope.createDefaultContentScannerService(
+        contentScanner: ContentScanner = FakeContentScanner(),
+        sessionCoroutineScope: CoroutineScope = backgroundScope,
+        coroutineDispatchers: CoroutineDispatchers = testCoroutineDispatchers(),
+    ): DefaultContentScannerService {
+        return DefaultContentScannerService(
+            contentScanner = contentScanner,
+            coroutineScope = sessionCoroutineScope,
+            coroutineDispatchers = coroutineDispatchers
+        )
+    }
+}
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.features.contentscanner.impl
+
+import com.google.common.truth.Truth.assertThat
+import io.element.android.libraries.matrix.api.scanner.ContentScanner
+import io.element.android.libraries.matrix.test.AN_EVENT_ID
+import io.element.android.libraries.matrix.test.scanner.FakeContentScanner
+import io.element.android.libraries.matrix.ui.media.contentvalidation.DefaultContentValidationState
+import io.element.android.libraries.matrix.ui.media.contentvalidation.NoopContentValidationState
+import org.junit.Test
+
+class DefaultEventContentValidationCacheTest {
+    @Test
+    fun `get returns NoopContentValidationState when ContentScanner is null`() {
+        // Given
+        val contentScanner: ContentScanner? = null
+        val cache = DefaultEventContentValidationCache(contentScanner)
+
+        // When
+        val state = cache[AN_EVENT_ID]
+
+        // Then
+        assertThat(state).isInstanceOf(NoopContentValidationState::class.java)
+    }
+
+    @Test
+    fun `get returns an actual DefaultContentValidationState when ContentScanner is not null`() {
+        // Given
+        val contentScanner: ContentScanner? = FakeContentScanner()
+        val cache = DefaultEventContentValidationCache(contentScanner)
+
+        // When
+        val state = cache[AN_EVENT_ID]
+
+        // Then
+        assertThat(state).isInstanceOf(DefaultContentValidationState::class.java)
+    }
+}
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package io.element.android.features.contentscanner.impl.di
+
+import com.google.common.truth.Truth.assertThat
+import io.element.android.features.contentscanner.impl.DefaultContentScannerService
+import io.element.android.libraries.matrix.api.scanner.ContentScanner
+import io.element.android.libraries.matrix.test.scanner.FakeContentScanner
+import io.element.android.tests.testutils.testCoroutineDispatchers
+import kotlinx.coroutines.test.runTest
+import org.junit.Test
+
+class ContentScannerModuleTest {
+    @Test
+    fun `ContentScannerModule provides NoopContentScannerService when ContentScanner is null`() = runTest {
+        // Given
+        val contentScanner: ContentScanner? = null
+        val sessionCoroutineScope = backgroundScope
+        val coroutineDispatchers = testCoroutineDispatchers()
+
+        // When
+        val contentScannerService = ContentScannerModule.providesContentScannerService(
+            contentScanner = contentScanner,
+            coroutineScope = sessionCoroutineScope,
+            coroutineDispatchers = coroutineDispatchers,
+        )
+
+        // Then
+        assertThat(contentScannerService).isInstanceOf(AlwaysValidContentScannerService::class.java)
+    }
+
+    @Test
+    fun `ContentScannerModule provides DefaultContentScannerService when ContentScanner is not null`() = runTest {
+        // Given
+        val contentScanner: ContentScanner? = FakeContentScanner()
+        val sessionCoroutineScope = backgroundScope
+        val coroutineDispatchers = testCoroutineDispatchers()
+
+        // When
+        val contentScannerService = ContentScannerModule.providesContentScannerService(
+            contentScanner = contentScanner,
+            coroutineScope = sessionCoroutineScope,
+            coroutineDispatchers = coroutineDispatchers,
+        )
+
+        // Then
+        assertThat(contentScannerService).isInstanceOf(DefaultContentScannerService::class.java)
+    }
+}