Index messages locally in an encrypted on-device store so encrypted rooms can be searched at all (#7249)

* Add a message search abstraction over the Rust SDK's local index

Wraps the SDK's search service so the rest of the app never sees MatrixRustSDK
types, following the existing convention in libraries/matrix.

- MessageSearch, MessageSearchService, MessageSearchResult and
  MessageSearchPaginationState describe the surface in the api module.
- RustMessageSearch and RustMessageSearchService implement it, with
  MessageSearchResultMapper and MessageSearchResultsProcessor turning SDK results
  into Kotlin data classes.
- TantivyQueryEscaper escapes the query so a user typing ":)" or "AND" searches
  for that literal text rather than tripping the tantivy parser, and joins every
  word as a Must clause.
- Fakes and unit tests cover the mapping, the processor and the escaper, the last
  checked against a real tantivy 0.26.1 parser.

* Attach the encrypted search index behind a disabled feature flag

Gives the client an on-device tantivy index encrypted with the session's client
secret, so encrypted rooms can be searched at all.

- The index store is attached at ClientBuilder time via withSearchIndexStore, in
  a search-index directory alongside the session, keyed on the client secret. A
  session without one gets no index.
- MatrixClient exposes isMessageSearchAvailable so callers gate on the real
  indexing capability rather than on the raw flag, and getCacheSize() counts the
  index directory so it stays visible in the settings figures.
- FeatureFlags.MessageSearch (feature.message_search) is off by default and
  isFinished = false. It is read once at ClientBuilder time, so develop stays
  releasable and nothing reaches a user.

* Simplify search index wiring following review

Tie search availability to the MessageSearch feature flag alone and let
the index be created without a client secret, encrypting it only when
the session has one, matching the SDK's SQLite stores. Make
getBaseClientBuilder take the flag as a required parameter and return
ClientBuilder directly again, dropping the BaseClientBuilder wrapper
and the mid-restore snapshot parameter. Check the flag through
FeatureFlagService in RustMatrixAuthenticationService instead of
caching it in a field, and revert the extra matrix_sdk_search tracing
target, which belongs in the SDK's defaults.
This commit is contained in:
hayyaksi
2026-07-30 12:17:40 +03:00
committed by GitHub
parent af9db20f4f
commit e6d4472c0c
24 changed files with 1353 additions and 8 deletions
@@ -81,6 +81,7 @@ import io.element.android.libraries.matrix.impl.roomdirectory.map
import io.element.android.libraries.matrix.impl.roomlist.RoomListFactory
import io.element.android.libraries.matrix.impl.roomlist.RustRoomListService
import io.element.android.libraries.matrix.impl.roomlist.roomOrNull
import io.element.android.libraries.matrix.impl.search.RustMessageSearchService
import io.element.android.libraries.matrix.impl.spaces.RustSpaceService
import io.element.android.libraries.matrix.impl.sync.RustSyncService
import io.element.android.libraries.matrix.impl.sync.map
@@ -155,6 +156,7 @@ class RustMatrixClient(
private val analyticsService: AnalyticsService,
private val workManagerScheduler: WorkManagerScheduler,
override val contentScanner: ContentScanner?,
override val isMessageSearchAvailable: Boolean,
) : MatrixClient {
override val sessionId: UserId = UserId(innerClient.userId())
override val deviceId: DeviceId = DeviceId(innerClient.deviceId())
@@ -194,6 +196,11 @@ class RustMatrixClient(
sessionDispatcher = sessionDispatcher,
)
override val messageSearchService = RustMessageSearchService(
client = innerClient,
sessionDispatcher = sessionDispatcher,
)
private val roomSyncSubscriber: RoomSyncSubscriber = RoomSyncSubscriber(innerRoomListService, dispatchers)
override val roomListService: RoomListService = RustRoomListService(
@@ -876,7 +883,11 @@ class RustMatrixClient(
File(sessionPaths.fileDirectory, fileName)
}.sumOf { file ->
file.length()
}
} +
// The message search index. Like the state database above it lives in the file
// directory and so survives a cache clear, but it can grow without bound and must
// not be invisible in the storage figures. Returns 0 when the index is disabled.
File(sessionPaths.fileDirectory, SEARCH_INDEX_DIRECTORY).getSizeOfFiles()
}
}
@@ -81,10 +81,23 @@ class RustMatrixClientFactory(
suspend fun create(sessionData: SessionData): RustMatrixClient = withContext(coroutineDispatchers.io) {
// This secret is called 'passphrase' for historical reasons, but it can be a raw key or an actual passphrase
val clientSecret = sessionData.passphrase?.let(ClientSecret::fromString)
val sessionPaths = sessionData.getSessionPaths()
val isMessageSearchAvailable = featureFlagService.isFeatureEnabled(FeatureFlags.MessageSearch)
val indexDirectory = File(sessionPaths.fileDirectory, SEARCH_INDEX_DIRECTORY)
if (!isMessageSearchAvailable && indexDirectory.exists()) {
// With search off, events reach the event cache unindexed and the index can never
// catch up on them. Delete it so the next enable rebuilds it from a state where
// coverage can actually be guaranteed, instead of resuming a silently stale index.
Timber.tag("RustMatrixClient").i("Message search is disabled, deleting the stale search index")
indexDirectory.deleteRecursively()
}
val client = getBaseClientBuilder(
sessionPaths = sessionData.getSessionPaths(),
sessionPaths = sessionPaths,
clientSecret = clientSecret,
slidingSyncType = ClientBuilderSlidingSync.Restored,
isMessageSearchAvailable = isMessageSearchAvailable,
)
.homeserverUrl(sessionData.homeserverUrl)
.username(sessionData.userId)
@@ -105,10 +118,14 @@ class RustMatrixClientFactory(
client.restoreSession(sessionData.toSession())
create(client, sessionData)
create(client, sessionData, isMessageSearchAvailable)
}
suspend fun create(client: Client, sessionData: SessionData): RustMatrixClient {
suspend fun create(
client: Client,
sessionData: SessionData,
isMessageSearchAvailable: Boolean,
): RustMatrixClient {
val (anonymizedAccessToken, anonymizedRefreshToken) = client.session().anonymizedTokens()
// Must be called before creating the sync service, timelines etc.
@@ -155,6 +172,7 @@ class RustMatrixClientFactory(
analyticsService = analyticsService,
workManagerScheduler = workManagerScheduler,
contentScanner = contentScanner,
isMessageSearchAvailable = isMessageSearchAvailable,
).also {
Timber.tag("RustMatrixClient").i("Creating Client with access token '$anonymizedAccessToken' and refresh token '$anonymizedRefreshToken'")
}
@@ -164,6 +182,7 @@ class RustMatrixClientFactory(
sessionPaths: SessionPaths,
clientSecret: ClientSecret?,
slidingSyncType: ClientBuilderSlidingSync,
isMessageSearchAvailable: Boolean,
): ClientBuilder {
return clientBuilderProvider.provide()
.run {
@@ -193,6 +212,18 @@ class RustMatrixClientFactory(
)
.enableShareHistoryOnInvite(true)
.threadsEnabled(featureFlagService.isFeatureEnabled(FeatureFlags.Threads), threadSubscriptions = false)
.run {
if (isMessageSearchAvailable) {
// The index is encrypted at rest with the same secret the SDK's SQLite stores
// use, or left unencrypted for sessions without one, matching those stores.
withSearchIndexStore(
path = File(sessionPaths.fileDirectory, SEARCH_INDEX_DIRECTORY).absolutePath,
password = clientSecret?.formattedAsString(),
)
} else {
this
}
}
.dmRoomDefinition(DmRoomDefinition.TWO_MEMBERS)
.requestConfig(
RequestConfig(
@@ -221,6 +252,12 @@ class RustMatrixClientFactory(
}
}
/**
* Directory holding the local message search index, under the session's file directory.
* Not the cache directory: the index is not disposable, and a cache wipe should not destroy it.
*/
internal const val SEARCH_INDEX_DIRECTORY = "search-index"
sealed interface ClientBuilderSlidingSync {
// The proxy will be supplied when restoring the Session.
data object Restored : ClientBuilderSlidingSync
@@ -16,6 +16,8 @@ import io.element.android.libraries.androidutils.crypto.ClientSecret
import io.element.android.libraries.core.coroutine.CoroutineDispatchers
import io.element.android.libraries.core.extensions.mapFailure
import io.element.android.libraries.core.extensions.runCatchingExceptions
import io.element.android.libraries.featureflag.api.FeatureFlagService
import io.element.android.libraries.featureflag.api.FeatureFlags
import io.element.android.libraries.matrix.api.MatrixClient
import io.element.android.libraries.matrix.api.auth.AuthenticationException
import io.element.android.libraries.matrix.api.auth.ElementClassicSession
@@ -69,6 +71,7 @@ class RustMatrixAuthenticationService(
private val secretGenerator: SecretGenerator,
private val oAuthConfigurationProvider: OAuthConfigurationProvider,
private val enterpriseService: EnterpriseService,
private val featureFlagService: FeatureFlagService,
) : MatrixAuthenticationService {
// Any existing Element Classic session that we want to try to import secrets from during login.
private var elementClassicSession: ElementClassicSession? = null
@@ -160,7 +163,7 @@ class RustMatrixAuthenticationService(
passphrase = pendingKey.formattedAsString(),
sessionPaths = currentSessionPaths,
)
val matrixClient = rustMatrixClientFactory.create(client, sessionData)
val matrixClient = rustMatrixClientFactory.create(client, sessionData, isMessageSearchAvailable())
newMatrixClientObservers.forEach { it.invoke(matrixClient) }
sessionStore.addSession(sessionData)
@@ -235,7 +238,7 @@ class RustMatrixAuthenticationService(
// We restore the client using the just retrieved session data
client.restoreSession(sessionData.toSession())
val matrixClient = rustMatrixClientFactory.create(client, sessionData)
val matrixClient = rustMatrixClientFactory.create(client, sessionData, isMessageSearchAvailable())
// We wait for the verification state to be known
matrixClient.waitForKnownVerificationState()
@@ -325,7 +328,7 @@ class RustMatrixAuthenticationService(
passphrase = pendingKey.formattedAsString(),
sessionPaths = currentSessionPaths,
)
val matrixClient = rustMatrixClientFactory.create(client, sessionData)
val matrixClient = rustMatrixClientFactory.create(client, sessionData, isMessageSearchAvailable())
matrixClient.waitForKnownVerificationState()
newMatrixClientObservers.forEach { it.invoke(matrixClient) }
@@ -390,7 +393,7 @@ class RustMatrixAuthenticationService(
passphrase = pendingKey.formattedAsString(),
sessionPaths = emptySessionPaths,
)
val matrixClient = rustMatrixClientFactory.create(client, sessionData)
val matrixClient = rustMatrixClientFactory.create(client, sessionData, isMessageSearchAvailable())
newMatrixClientObservers.forEach { it.invoke(matrixClient) }
sessionStore.addSession(sessionData)
@@ -422,6 +425,7 @@ class RustMatrixAuthenticationService(
sessionPaths = sessionPaths,
clientSecret = pendingKey,
slidingSyncType = ClientBuilderSlidingSync.Discovered,
isMessageSearchAvailable = isMessageSearchAvailable(),
)
.config()
.build()
@@ -454,6 +458,7 @@ class RustMatrixAuthenticationService(
sessionPaths = sessionPaths,
clientSecret = pendingKey,
slidingSyncType = ClientBuilderSlidingSync.Discovered,
isMessageSearchAvailable = isMessageSearchAvailable(),
)
.serverNameOrHomeserverUrl(baseUrlOrServerName)
.build()
@@ -464,6 +469,9 @@ class RustMatrixAuthenticationService(
currentClient = null
}
private suspend fun isMessageSearchAvailable(): Boolean =
featureFlagService.isFeatureEnabled(FeatureFlags.MessageSearch)
private suspend fun MatrixClient.waitForKnownVerificationState() {
withTimeoutOrNull(10.seconds) {
Timber.d("Waiting for a known verification status...")
@@ -0,0 +1,39 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.impl.search
import io.element.android.libraries.matrix.api.core.EventId
import io.element.android.libraries.matrix.api.core.RoomId
import io.element.android.libraries.matrix.api.core.UserId
import io.element.android.libraries.matrix.api.search.MessageSearchResult
import io.element.android.libraries.matrix.impl.timeline.item.event.TimelineEventContentMapper
import io.element.android.libraries.matrix.impl.timeline.item.event.map
import org.matrix.rustcomponents.sdk.SearchServiceResult
class MessageSearchResultMapper(
private val contentMapper: TimelineEventContentMapper = TimelineEventContentMapper(),
) {
/**
* Note: [TimelineEventContentMapper.map] consumes (and destroys) the content, so each result
* must be mapped exactly once.
*/
fun map(result: SearchServiceResult): MessageSearchResult {
// Exhaustive on purpose, with no `else`: a new SDK result kind must fail compilation here
// rather than being silently dropped, which would desync our list from the SDK's indices.
return when (result) {
is SearchServiceResult.Message -> MessageSearchResult(
roomId = RoomId(result.roomId),
eventId = EventId(result.result.eventId),
senderId = UserId(result.result.sender),
senderProfile = result.result.senderProfile.map(),
content = contentMapper.map(result.result.content),
timestamp = result.result.timestamp.toLong(),
)
}
}
}
@@ -0,0 +1,84 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.impl.search
import io.element.android.libraries.matrix.api.search.MessageSearchResult
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import org.matrix.rustcomponents.sdk.SearchServiceResultsUpdate
import kotlin.coroutines.CoroutineContext
/**
* Applies the SDK's diff stream to a local list of search results.
*
* The local list is kept strictly index-parallel to the SDK's: every result is mapped, nothing is
* filtered or dropped, so the positional updates ([SearchServiceResultsUpdate.Insert],
* [SearchServiceResultsUpdate.Set], [SearchServiceResultsUpdate.Remove],
* [SearchServiceResultsUpdate.Truncate]) always address the slot the SDK meant. Filtering for
* display, if ever needed, belongs at render time — not here.
*/
class MessageSearchResultsProcessor(
private val results: MutableSharedFlow<List<MessageSearchResult>>,
private val coroutineContext: CoroutineContext,
private val mapper: MessageSearchResultMapper = MessageSearchResultMapper(),
) {
private val mutex = Mutex()
suspend fun postUpdates(updates: List<SearchServiceResultsUpdate>) = withContext(coroutineContext) {
mutex.withLock {
val current = results.replayCache.lastOrNull().orEmpty().toMutableList()
updates.forEach { update ->
current.applyUpdate(update)
}
results.emit(current)
}
}
private fun MutableList<MessageSearchResult>.applyUpdate(update: SearchServiceResultsUpdate) {
// Exhaustive on purpose, with no `else`: a future SDK bump adding a variant must fail
// compilation rather than silently dropping updates and corrupting the list.
when (update) {
is SearchServiceResultsUpdate.Append -> {
addAll(update.values.map(mapper::map))
}
is SearchServiceResultsUpdate.PushBack -> {
add(mapper.map(update.value))
}
is SearchServiceResultsUpdate.PushFront -> {
add(0, mapper.map(update.value))
}
is SearchServiceResultsUpdate.Set -> {
this[update.index.toInt()] = mapper.map(update.value)
}
is SearchServiceResultsUpdate.Insert -> {
add(update.index.toInt(), mapper.map(update.value))
}
is SearchServiceResultsUpdate.Remove -> {
removeAt(update.index.toInt())
}
is SearchServiceResultsUpdate.Truncate -> {
subList(update.length.toInt(), size).clear()
}
is SearchServiceResultsUpdate.Reset -> {
clear()
addAll(update.values.map(mapper::map))
}
SearchServiceResultsUpdate.PopBack -> {
removeLastOrNull()
}
SearchServiceResultsUpdate.PopFront -> {
removeFirstOrNull()
}
SearchServiceResultsUpdate.Clear -> {
clear()
}
}
}
}
@@ -0,0 +1,153 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.impl.search
import io.element.android.libraries.core.extensions.runCatchingExceptions
import io.element.android.libraries.matrix.api.core.RoomId
import io.element.android.libraries.matrix.api.search.MessageSearch
import io.element.android.libraries.matrix.api.search.MessageSearchPaginationState
import io.element.android.libraries.matrix.api.search.MessageSearchResult
import io.element.android.libraries.matrix.impl.util.TaskHandleBag
import kotlinx.collections.immutable.ImmutableList
import kotlinx.collections.immutable.persistentListOf
import kotlinx.collections.immutable.toImmutableList
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.SharingStarted
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.consumeAsFlow
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.stateIn
import kotlinx.coroutines.job
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import org.matrix.rustcomponents.sdk.SearchServiceInterface
import org.matrix.rustcomponents.sdk.SearchServicePaginationStateListener
import org.matrix.rustcomponents.sdk.SearchServiceResultsListener
import org.matrix.rustcomponents.sdk.SearchServiceResultsUpdate
import uniffi.matrix_sdk_ui.SearchServicePaginationState
/**
* Wraps the SDK's stateful [SearchServiceInterface] as a single search cursor.
*
* Lifecycle: the caller's [CoroutineScope] owns this instance. When that scope completes, both
* subscription [org.matrix.rustcomponents.sdk.TaskHandle]s are cancelled and [onClose] releases the
* underlying service — the handles must be retained until then, or the Rust side drops the
* subscription.
*
* PII: search queries and message bodies are never logged.
*/
class RustMessageSearch(
private val inner: SearchServiceInterface,
private val onClose: () -> Unit,
scope: CoroutineScope,
private val dispatcher: CoroutineDispatcher,
private val roomId: RoomId? = null,
) : MessageSearch {
private val innerResults = MutableStateFlow<List<MessageSearchResult>>(emptyList())
private val processor = MessageSearchResultsProcessor(
results = innerResults,
coroutineContext = dispatcher,
)
private val taskHandles = TaskHandleBag()
private val subscriptionMutex = Mutex()
private var subscribedToResults = false
/**
* The SDK listener is not a suspend function, so batches are handed to a single consumer
* coroutine. An unlimited channel keeps them strictly in arrival order — applying diffs out of
* order would corrupt the list just as surely as dropping them.
*/
private val updates = Channel<List<SearchServiceResultsUpdate>>(Channel.UNLIMITED)
override val results: StateFlow<ImmutableList<MessageSearchResult>> = innerResults
.map { results ->
// Room scoping is applied HERE, at the exposure boundary, and nowhere else.
// [innerResults] must stay index-parallel to the SDK's own list, because the positional
// diffs (Insert/Set/Remove/Truncate) address the SDK's indices. Filtering inside
// MessageSearchResultsProcessor instead is precisely the element-x-ios desync bug.
when (roomId) {
null -> results.toImmutableList()
else -> results.filter { it.roomId == roomId }.toImmutableList()
}
}
.stateIn(scope, SharingStarted.Eagerly, persistentListOf())
private val mutablePaginationState = MutableStateFlow(inner.paginationState().map())
override val paginationState: StateFlow<MessageSearchPaginationState> = mutablePaginationState.asStateFlow()
init {
updates.consumeAsFlow()
.onEach { processor.postUpdates(it) }
.launchIn(scope)
taskHandles += inner.subscribeToPaginationStateUpdates(
object : SearchServicePaginationStateListener {
override fun onUpdate(paginationState: SearchServicePaginationState) {
mutablePaginationState.value = paginationState.map()
}
}
)
scope.coroutineContext.job.invokeOnCompletion {
taskHandles.dispose()
updates.close()
onClose()
}
}
override suspend fun setQuery(query: String): Result<Unit> = withContext(dispatcher) {
runCatchingExceptions {
// Subscribe lazily, so a search screen that is opened but never used costs nothing.
subscribeToResultsIfNeeded()
// The SDK contract for setQuery is "clears the current results, restarts pagination from
// scratch". Whatever endReached we are holding describes the PREVIOUS query — or, on the
// very first search, a cursor that was never queried at all and reports endReached=true
// because there is nothing to page through yet. Carrying it over makes every search look
// finished before it starts, so callers never paginate and an empty result is reported as
// a definitive "no results". Reset before handing the query over; the subscription
// corrects this the moment the SDK reports its real state.
mutablePaginationState.value = MessageSearchPaginationState.Idle(endReached = false)
// The SDK parses this with tantivy's strict query parser, where `:` and friends are
// syntax rather than text — a pasted URL fails the whole search. Escape so the query is
// searched literally. The caller keeps the raw string; only the SDK sees the escaped one.
inner.setQuery(query.escapeForTantivy())
}
}
override suspend fun paginate(): Result<Unit> = withContext(dispatcher) {
runCatchingExceptions {
inner.paginate()
}
}
private suspend fun subscribeToResultsIfNeeded() {
subscriptionMutex.withLock {
if (subscribedToResults) return@withLock
taskHandles += inner.subscribeToResults(
object : SearchServiceResultsListener {
override fun onUpdate(updates: List<SearchServiceResultsUpdate>) {
this@RustMessageSearch.updates.trySend(updates)
}
}
)
subscribedToResults = true
}
}
}
internal fun SearchServicePaginationState.map(): MessageSearchPaginationState = when (this) {
is SearchServicePaginationState.Idle -> MessageSearchPaginationState.Idle(endReached = endReached)
SearchServicePaginationState.Loading -> MessageSearchPaginationState.Loading
}
@@ -0,0 +1,33 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.impl.search
import io.element.android.libraries.matrix.api.core.RoomId
import io.element.android.libraries.matrix.api.search.MessageSearch
import io.element.android.libraries.matrix.api.search.MessageSearchService
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.CoroutineScope
import org.matrix.rustcomponents.sdk.Client
class RustMessageSearchService(
private val client: Client,
private val sessionDispatcher: CoroutineDispatcher,
) : MessageSearchService {
override fun createMessageSearch(scope: CoroutineScope, roomId: RoomId?): MessageSearch {
// Each searchService() call mints a new Rust object, so this instance owns the one it
// creates and closes it when the caller's scope completes.
val searchService = client.searchService()
return RustMessageSearch(
inner = searchService,
onClose = { searchService.close() },
scope = scope,
dispatcher = sessionDispatcher,
roomId = roomId,
)
}
}
@@ -0,0 +1,85 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.impl.search
/**
* Characters that terminate a bare word wherever they appear in it. Escaping them keeps the token a
* single literal word. `\` is included so a backslash the user typed cannot escape one of ours, and
* `*` is the prefix/all-documents operator rather than a word breaker.
*/
private val ESCAPED_ANYWHERE = charArrayOf('\\', '^', '`', ':', '{', '}', '"', '\'', '[', ']', '(', ')', '*')
/**
* Characters that are operators only in the first position of a term: `-`/`+` occurrence, `/` regex,
* `>`/`<` elastic range, `!` field separator, `~` slop. Inside a word they are ordinary text, so
* escaping them there would be over-broad — `C++` and `1/2/2024` must survive untouched.
*/
private val ESCAPED_WHEN_LEADING = charArrayOf('-', '+', '/', '>', '<', '!', '~')
/**
* Bare-word operators, neutralised so the query stays literal text end to end: `a AND b` searches
* for the three words rather than silently switching to a boolean AND. Mid-query these change
* meaning rather than failing; only a dangling operator is a syntax error.
*/
private val OPERATOR_WORDS = setOf("AND", "OR", "NOT", "IN", "TO")
private val WHITESPACE_REGEX = Regex("\\s+")
private const val ESCAPE_HEADROOM = 8
/**
* Escapes a user-typed search string so tantivy's query parser reads it as literal text, and makes
* every word of it mandatory.
*
* **Escaping.** `matrix-sdk-search` hands the query to the **strict** `QueryParser::parse_query` with
* `body` as the only default field, so an unescaped `:` sends tantivy looking for a field named
* `https` and the whole search fails — pasting a link finds nothing at all. Unbalanced quotes and
* brackets, a leading `-`, and elastic ranges like `>5` fail the same way, and the error arrives as
* an opaque `ClientError` string we cannot tell apart from an I/O failure.
*
* Escaping is per token rather than wrapping the whole query in quotes: whitespace stays the term
* separator, and only a token that actually contained an operator changes shape. Wrapping instead
* would turn `design review notes` into one ordered phrase and quietly break every ordinary search.
*
* **Conjunction.** tantivy's parser is OR-by-default and the FFI takes a bare string, so
* `set_conjunction_by_default()` is unreachable from here — `check the photo` matched every message
* containing `the`, and buried the wanted one under hundreds of unrelated hits ranked by relevance.
* Prefixing each token with `+` makes it a Must clause, which parses to exactly the query
* `set_conjunction_by_default()` would have built.
*
* The `+` goes on **after** escaping, never before: `+` is escaped in leading position, so prefixing
* first would hand the escaper its own operator and produce a literal `\+`. That parses cleanly and
* silently restores OR, so it cannot be caught by anything but an assertion on the exact string.
*
* The costs, stated plainly: deliberate `"phrase"`, `-exclude` and boolean syntax stop working as
* operators; and a query is now only as good as its worst word — one typo, or one word the user
* misremembers, empties the result set where before it degraded to partial matches.
*
* Pinned to tantivy 0.26.1 grammar. PII: the query is never logged.
*/
internal fun String.escapeForTantivy(): String =
split(WHITESPACE_REGEX)
.filter { it.isNotEmpty() }
.joinToString(separator = " ") { token ->
// tantivy's default tokenizer splits on non-alphanumerics, so a token like `:)` or an
// emoji indexes no terms at all. Required, such a token could only ever subtract, and a
// single emoji beside real words would silently empty the results. Optional, it is
// inert — which is what it was before conjunctions too.
if (token.any(Char::isLetterOrDigit)) "+${token.escapeToken()}" else token.escapeToken()
}
private fun String.escapeToken(): String = buildString(length + ESCAPE_HEADROOM) {
if (this@escapeToken in OPERATOR_WORDS) append('\\')
this@escapeToken.forEachIndexed { index, char ->
// One forward pass: chained replace() calls would re-escape the backslashes they just added.
if (char in ESCAPED_ANYWHERE || index == 0 && char in ESCAPED_WHEN_LEADING) {
append('\\')
}
append(char)
}
}
@@ -160,5 +160,6 @@ class RustMatrixClientTest {
analyticsService = FakeAnalyticsService(),
workManagerScheduler = FakeWorkManagerScheduler(submitLambda = {}),
contentScanner = FakeContentScanner(),
isMessageSearchAvailable = false,
)
}
@@ -11,6 +11,7 @@ package io.element.android.libraries.matrix.impl.auth
import com.google.common.truth.Truth.assertThat
import io.element.android.features.enterprise.api.EnterpriseService
import io.element.android.features.enterprise.test.FakeEnterpriseService
import io.element.android.libraries.featureflag.test.FakeFeatureFlagService
import io.element.android.libraries.matrix.impl.ClientBuilderProvider
import io.element.android.libraries.matrix.impl.FakeClientBuilderProvider
import io.element.android.libraries.matrix.impl.createRustMatrixClientFactory
@@ -72,6 +73,7 @@ class RustMatrixAuthenticationServiceTest {
oAuthRedirectUrlProvider = FakeOAuthRedirectUrlProvider(),
),
enterpriseService = enterpriseService,
featureFlagService = FakeFeatureFlagService(),
)
}
}
@@ -0,0 +1,31 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.impl.fixtures.factories
import io.element.android.libraries.matrix.test.A_ROOM_ID
import io.element.android.libraries.matrix.test.A_USER_ID
import org.matrix.rustcomponents.sdk.MessageSearchResult
import org.matrix.rustcomponents.sdk.ProfileDetails
import org.matrix.rustcomponents.sdk.SearchServiceResult
internal fun aRustSearchServiceResult(
eventId: String,
roomId: String = A_ROOM_ID.value,
sender: String = A_USER_ID.value,
body: String = "Hello",
timestamp: ULong = 0uL,
) = SearchServiceResult.Message(
roomId = roomId,
result = MessageSearchResult(
eventId = eventId,
sender = sender,
senderProfile = ProfileDetails.Unavailable,
content = aRustTimelineItemContentMsgLike(body = body),
timestamp = timestamp,
),
)
@@ -0,0 +1,45 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.impl.fixtures.fakes
import org.matrix.rustcomponents.sdk.NoHandle
import org.matrix.rustcomponents.sdk.SearchService
import org.matrix.rustcomponents.sdk.SearchServicePaginationStateListener
import org.matrix.rustcomponents.sdk.SearchServiceResultsListener
import org.matrix.rustcomponents.sdk.TaskHandle
import uniffi.matrix_sdk_ui.SearchServicePaginationState
class FakeFfiSearchService(
private val initialPaginationState: SearchServicePaginationState = SearchServicePaginationState.Idle(endReached = false),
private val setQueryLambda: (String) -> Unit = {},
private val paginateLambda: () -> Unit = {},
) : SearchService(NoHandle) {
var resultsListener: SearchServiceResultsListener? = null
private set
var paginationStateListener: SearchServicePaginationStateListener? = null
private set
var subscribeToResultsCallCount = 0
private set
override suspend fun paginate() = paginateLambda()
override fun paginationState(): SearchServicePaginationState = initialPaginationState
override suspend fun setQuery(query: String) = setQueryLambda(query)
override fun subscribeToPaginationStateUpdates(listener: SearchServicePaginationStateListener): TaskHandle {
paginationStateListener = listener
return FakeFfiTaskHandle()
}
override suspend fun subscribeToResults(listener: SearchServiceResultsListener): TaskHandle {
subscribeToResultsCallCount++
resultsListener = listener
return FakeFfiTaskHandle()
}
}
@@ -0,0 +1,176 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.impl.search
import com.google.common.truth.Truth.assertThat
import io.element.android.libraries.matrix.api.search.MessageSearchResult
import io.element.android.libraries.matrix.impl.fixtures.factories.aRustSearchServiceResult
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.test.runTest
import org.junit.Test
import org.matrix.rustcomponents.sdk.SearchServiceResultsUpdate
class MessageSearchResultsProcessorTest {
private val results = MutableStateFlow<List<MessageSearchResult>>(emptyList())
private fun createProcessor() = MessageSearchResultsProcessor(
results = results,
coroutineContext = kotlin.coroutines.EmptyCoroutineContext,
)
private fun eventIds() = results.value.map { it.eventId.value }
@Test
fun `Append adds entries at the end of the list`() = runTest {
val processor = createProcessor()
processor.postUpdates(listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1"), aRustSearchServiceResult("\$2")))))
assertThat(eventIds()).isEqualTo(listOf("\$1", "\$2"))
}
@Test
fun `PushBack adds an entry at the end of the list`() = runTest {
val processor = createProcessor()
processor.postUpdates(listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1")))))
processor.postUpdates(listOf(SearchServiceResultsUpdate.PushBack(aRustSearchServiceResult("\$2"))))
assertThat(eventIds()).isEqualTo(listOf("\$1", "\$2"))
}
@Test
fun `PushFront inserts an entry at the start of the list`() = runTest {
val processor = createProcessor()
processor.postUpdates(listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1")))))
processor.postUpdates(listOf(SearchServiceResultsUpdate.PushFront(aRustSearchServiceResult("\$0"))))
assertThat(eventIds()).isEqualTo(listOf("\$0", "\$1"))
}
@Test
fun `PopBack removes the last entry`() = runTest {
val processor = createProcessor()
processor.postUpdates(listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1"), aRustSearchServiceResult("\$2")))))
processor.postUpdates(listOf(SearchServiceResultsUpdate.PopBack))
assertThat(eventIds()).isEqualTo(listOf("\$1"))
}
@Test
fun `PopFront removes the first entry`() = runTest {
val processor = createProcessor()
processor.postUpdates(listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1"), aRustSearchServiceResult("\$2")))))
processor.postUpdates(listOf(SearchServiceResultsUpdate.PopFront))
assertThat(eventIds()).isEqualTo(listOf("\$2"))
}
@Test
fun `Insert puts an entry at the given index`() = runTest {
val processor = createProcessor()
processor.postUpdates(listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1"), aRustSearchServiceResult("\$3")))))
processor.postUpdates(listOf(SearchServiceResultsUpdate.Insert(1u, aRustSearchServiceResult("\$2"))))
assertThat(eventIds()).isEqualTo(listOf("\$1", "\$2", "\$3"))
}
@Test
fun `Set replaces the entry at the given index`() = runTest {
val processor = createProcessor()
processor.postUpdates(listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1"), aRustSearchServiceResult("\$2")))))
processor.postUpdates(listOf(SearchServiceResultsUpdate.Set(1u, aRustSearchServiceResult("\$9"))))
assertThat(eventIds()).isEqualTo(listOf("\$1", "\$9"))
}
@Test
fun `Remove drops the entry at the given index`() = runTest {
val processor = createProcessor()
processor.postUpdates(
listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1"), aRustSearchServiceResult("\$2"), aRustSearchServiceResult("\$3"))))
)
processor.postUpdates(listOf(SearchServiceResultsUpdate.Remove(1u)))
assertThat(eventIds()).isEqualTo(listOf("\$1", "\$3"))
}
@Test
fun `Truncate keeps only the first N entries`() = runTest {
val processor = createProcessor()
processor.postUpdates(
listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1"), aRustSearchServiceResult("\$2"), aRustSearchServiceResult("\$3"))))
)
processor.postUpdates(listOf(SearchServiceResultsUpdate.Truncate(2u)))
assertThat(eventIds()).isEqualTo(listOf("\$1", "\$2"))
}
@Test
fun `Clear empties the list`() = runTest {
val processor = createProcessor()
processor.postUpdates(listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1")))))
processor.postUpdates(listOf(SearchServiceResultsUpdate.Clear))
assertThat(eventIds()).isEmpty()
}
@Test
fun `Reset replaces the whole list`() = runTest {
val processor = createProcessor()
processor.postUpdates(listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1"), aRustSearchServiceResult("\$2")))))
processor.postUpdates(listOf(SearchServiceResultsUpdate.Reset(listOf(aRustSearchServiceResult("\$7")))))
assertThat(eventIds()).isEqualTo(listOf("\$7"))
}
/**
* Regression test for the index-desync bug present in the element-x-ios bridge, where positional
* updates from the SDK were applied to a list that had already been filtered, so every index
* landed in the wrong slot. Our list must stay index-parallel to the SDK's throughout a batch.
*/
@Test
fun `positional updates in one batch stay index-parallel with the SDK`() = runTest {
val processor = createProcessor()
processor.postUpdates(
listOf(
SearchServiceResultsUpdate.Append(
listOf(aRustSearchServiceResult("\$1"), aRustSearchServiceResult("\$2"), aRustSearchServiceResult("\$3"))
),
SearchServiceResultsUpdate.Insert(1u, aRustSearchServiceResult("\$b")),
SearchServiceResultsUpdate.Remove(0u),
SearchServiceResultsUpdate.Set(0u, aRustSearchServiceResult("\$z")),
SearchServiceResultsUpdate.PushFront(aRustSearchServiceResult("\$a")),
SearchServiceResultsUpdate.Truncate(3u),
)
)
// Append -> [1, 2, 3]
// Insert(1,b) -> [1, b, 2, 3]
// Remove(0) -> [b, 2, 3]
// Set(0,z) -> [z, 2, 3]
// PushFront(a)-> [a, z, 2, 3]
// Truncate(3) -> [a, z, 2]
assertThat(eventIds()).isEqualTo(listOf("\$a", "\$z", "\$2"))
}
@Test
fun `a result carries roomId, sender and timestamp through the mapping`() = runTest {
val processor = createProcessor()
processor.postUpdates(
listOf(
SearchServiceResultsUpdate.Append(
listOf(aRustSearchServiceResult(eventId = "\$1", roomId = "!room:server", sender = "@bob:server", timestamp = 1234uL))
)
)
)
val result = results.value.single()
assertThat(result.roomId.value).isEqualTo("!room:server")
assertThat(result.senderId.value).isEqualTo("@bob:server")
assertThat(result.timestamp).isEqualTo(1234L)
}
}
@@ -0,0 +1,204 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.impl.search
import com.google.common.truth.Truth.assertThat
import io.element.android.libraries.matrix.api.search.MessageSearchPaginationState
import io.element.android.libraries.matrix.impl.fixtures.factories.aRustSearchServiceResult
import io.element.android.libraries.matrix.impl.fixtures.fakes.FakeFfiSearchService
import io.element.android.libraries.matrix.test.A_ROOM_ID
import io.element.android.libraries.matrix.test.A_ROOM_ID_2
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.cancel
import kotlinx.coroutines.test.StandardTestDispatcher
import kotlinx.coroutines.test.advanceUntilIdle
import kotlinx.coroutines.test.runTest
import org.junit.Test
import org.matrix.rustcomponents.sdk.SearchServiceResultsUpdate
import uniffi.matrix_sdk_ui.SearchServicePaginationState
class RustMessageSearchTest {
@Test
fun `paginationState is seeded from the SDK and reflects later updates`() = runTest {
val dispatcher = StandardTestDispatcher(testScheduler)
val scope = CoroutineScope(dispatcher)
val service = FakeFfiSearchService(initialPaginationState = SearchServicePaginationState.Idle(endReached = true))
val search = RustMessageSearch(inner = service, onClose = {}, scope = scope, dispatcher = dispatcher)
assertThat(search.paginationState.value).isEqualTo(MessageSearchPaginationState.Idle(endReached = true))
service.paginationStateListener!!.onUpdate(SearchServicePaginationState.Loading)
assertThat(search.paginationState.value).isEqualTo(MessageSearchPaginationState.Loading)
scope.cancel()
}
@Test
fun `setQuery failure surfaces as Result failure without throwing`() = runTest {
val dispatcher = StandardTestDispatcher(testScheduler)
val scope = CoroutineScope(dispatcher)
val service = FakeFfiSearchService(setQueryLambda = { error("boom") })
val search = RustMessageSearch(inner = service, onClose = {}, scope = scope, dispatcher = dispatcher)
val result = search.setQuery("hello")
assertThat(result.isFailure).isTrue()
scope.cancel()
}
@Test
fun `the query handed to the SDK is escaped and made conjunctive for tantivy`() = runTest {
val dispatcher = StandardTestDispatcher(testScheduler)
val scope = CoroutineScope(dispatcher)
var queryReceived: String? = null
val service = FakeFfiSearchService(setQueryLambda = { queryReceived = it })
val search = RustMessageSearch(inner = service, onClose = {}, scope = scope, dispatcher = dispatcher)
// Unescaped, tantivy reads `https` as a field name and fails the entire search. The `+`
// makes the term mandatory: without it tantivy is OR-by-default and a multi-word query
// matches anything containing any one of its words.
search.setQuery("https://github.com/foo")
assertThat(queryReceived).isEqualTo("+https\\://github.com/foo")
scope.cancel()
}
@Test
fun `paginate failure surfaces as Result failure without throwing`() = runTest {
val dispatcher = StandardTestDispatcher(testScheduler)
val scope = CoroutineScope(dispatcher)
val service = FakeFfiSearchService(paginateLambda = { error("boom") })
val search = RustMessageSearch(inner = service, onClose = {}, scope = scope, dispatcher = dispatcher)
val result = search.paginate()
assertThat(result.isFailure).isTrue()
scope.cancel()
}
@Test
fun `results are not subscribed to until the first setQuery, and only once`() = runTest {
val dispatcher = StandardTestDispatcher(testScheduler)
val scope = CoroutineScope(dispatcher)
val service = FakeFfiSearchService()
val search = RustMessageSearch(inner = service, onClose = {}, scope = scope, dispatcher = dispatcher)
assertThat(service.subscribeToResultsCallCount).isEqualTo(0)
search.setQuery("hello")
assertThat(service.subscribeToResultsCallCount).isEqualTo(1)
search.setQuery("world")
assertThat(service.subscribeToResultsCallCount).isEqualTo(1)
scope.cancel()
}
@Test
fun `updates pushed by the SDK reach the results flow`() = runTest {
val dispatcher = StandardTestDispatcher(testScheduler)
val scope = CoroutineScope(dispatcher)
val service = FakeFfiSearchService()
val search = RustMessageSearch(inner = service, onClose = {}, scope = scope, dispatcher = dispatcher)
search.setQuery("hello")
service.resultsListener!!.onUpdate(
listOf(SearchServiceResultsUpdate.Append(listOf(aRustSearchServiceResult("\$1"), aRustSearchServiceResult("\$2"))))
)
advanceUntilIdle()
assertThat(search.results.value.map { it.eventId.value }).isEqualTo(listOf("\$1", "\$2"))
scope.cancel()
}
@Test
fun `a room filter hides other rooms without disturbing SDK indexing`() = runTest {
val dispatcher = StandardTestDispatcher(testScheduler)
val scope = CoroutineScope(dispatcher)
val service = FakeFfiSearchService()
val search = RustMessageSearch(inner = service, onClose = {}, scope = scope, dispatcher = dispatcher, roomId = A_ROOM_ID)
search.setQuery("hello")
// SDK list: [$1@room1, $2@room2, $3@room1, $4@room2]
service.resultsListener!!.onUpdate(
listOf(
SearchServiceResultsUpdate.Append(
listOf(
aRustSearchServiceResult("\$1", roomId = A_ROOM_ID.value),
aRustSearchServiceResult("\$2", roomId = A_ROOM_ID_2.value),
aRustSearchServiceResult("\$3", roomId = A_ROOM_ID.value),
aRustSearchServiceResult("\$4", roomId = A_ROOM_ID_2.value),
)
)
)
)
advanceUntilIdle()
assertThat(search.results.value.map { it.eventId.value }).isEqualTo(listOf("\$1", "\$3"))
// Index 2 in the SDK's list is $3. Had the filter been applied inside the processor, the
// filtered list would be [$1, $3] and index 2 would be out of bounds — the iOS bug.
service.resultsListener!!.onUpdate(
listOf(SearchServiceResultsUpdate.Set(index = 2u, value = aRustSearchServiceResult("\$3edited", roomId = A_ROOM_ID.value)))
)
advanceUntilIdle()
assertThat(search.results.value.map { it.eventId.value }).isEqualTo(listOf("\$1", "\$3edited"))
// Removing index 1 ($2, a room we filter out) must not shift anything the user can see.
service.resultsListener!!.onUpdate(listOf(SearchServiceResultsUpdate.Remove(index = 1u)))
advanceUntilIdle()
assertThat(search.results.value.map { it.eventId.value }).isEqualTo(listOf("\$1", "\$3edited"))
scope.cancel()
}
@Test
fun `no room filter exposes every room`() = runTest {
val dispatcher = StandardTestDispatcher(testScheduler)
val scope = CoroutineScope(dispatcher)
val service = FakeFfiSearchService()
val search = RustMessageSearch(inner = service, onClose = {}, scope = scope, dispatcher = dispatcher, roomId = null)
search.setQuery("hello")
service.resultsListener!!.onUpdate(
listOf(
SearchServiceResultsUpdate.Append(
listOf(
aRustSearchServiceResult("\$1", roomId = A_ROOM_ID.value),
aRustSearchServiceResult("\$2", roomId = A_ROOM_ID_2.value),
)
)
)
)
advanceUntilIdle()
assertThat(search.results.value.map { it.eventId.value }).isEqualTo(listOf("\$1", "\$2"))
scope.cancel()
}
@Test
fun `cancelling the scope closes the underlying service`() = runTest {
val dispatcher = StandardTestDispatcher(testScheduler)
val scope = CoroutineScope(dispatcher)
var closed = false
val service = FakeFfiSearchService()
RustMessageSearch(inner = service, onClose = { closed = true }, scope = scope, dispatcher = dispatcher)
assertThat(closed).isFalse()
scope.cancel()
advanceUntilIdle()
assertThat(closed).isTrue()
}
}
@@ -0,0 +1,169 @@
/*
* Copyright (c) 2026 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.matrix.impl.search
import com.google.common.truth.Truth.assertThat
import org.junit.Test
/**
* Expectations here were checked against tantivy 0.26.1's own `QueryParser` — the version
* `matrix-sdk-search` compiles against — not inferred from the grammar. Each case notes the query it
* parses to, because "does not throw" is not the same as "searches for what the user typed".
*/
class TantivyQueryEscaperTest {
@Test
fun `an ordinary multi-word query becomes a conjunction`() {
// The headline behaviour. Without the `+`s tantivy is OR-by-default, so searching
// `check the photo` returned every message containing `the` — measured on device: a list of
// unrelated hits ranked by relevance, with the message actually wanted nowhere in it.
// Each `+` is a Must clause, which is what `set_conjunction_by_default()` would produce if
// the FFI let us call it.
assertThat("design review notes".escapeForTantivy()).isEqualTo("+design +review +notes")
}
@Test
fun `a single word query is a conjunction of one`() {
assertThat("photo".escapeForTantivy()).isEqualTo("+photo")
}
@Test
fun `the plus is prepended after escaping, never before`() {
// Load-bearing and silent when wrong: `-` is escaped only in leading position, so prefixing
// first would hand the escaper our own `+` to escape and produce `\+\-cats` — a literal
// plus, no Must clause, no parse error, and OR behaviour restored without a single failing
// assertion anywhere.
assertThat("-cats".escapeForTantivy()).isEqualTo("+\\-cats")
}
@Test
fun `a pasted URL no longer parses as a field lookup`() {
// The bug: unescaped, tantivy resolves `https` as a field name and fails the WHOLE search
// with FieldDoesNotExist. Only the colon needs escaping; the slashes are not leading.
assertThat("https://github.com/foo".escapeForTantivy()).isEqualTo("+https\\://github.com/foo")
}
@Test
fun `a real indexed field name is escaped too`() {
// `date` IS in the schema, as a DATE field, so this fails differently and more confusingly:
// DateFormatError rather than FieldDoesNotExist. The fix is not only about unknown fields.
assertThat("date:2020".escapeForTantivy()).isEqualTo("+date\\:2020")
}
@Test
fun `a matrix id is escaped`() {
// The second most likely paste after a URL. `@` and `.` are ordinary characters.
assertThat("@alice:matrix.org".escapeForTantivy()).isEqualTo("+@alice\\:matrix.org")
}
@Test
fun `a colon separated from its field name by a space is still escaped`() {
// The grammar allows whitespace between a field name and its colon, so `time: 12` would also
// be read as a field access. Per-token escaping defeats the spaced form as well.
assertThat("time: 12:30".escapeForTantivy()).isEqualTo("+time\\: +12\\:30")
}
@Test
fun `an elastic range operator is escaped`() {
// A genuine second failure class the desktop colon-only fix would have missed: unescaped,
// `>5` fails with UnsupportedQuery("Range query need to target a specific field.").
assertThat(">5".escapeForTantivy()).isEqualTo("+\\>5")
}
@Test
fun `quotes are escaped, which deliberately drops phrase search`() {
// Documents the cost: a typed phrase search becomes two AND'd terms. Accepted because a
// MISbalanced quote is a hard failure and the UI never advertised phrase syntax. Since the
// change to conjunctions this is much closer to a phrase search than it used to be — the
// words must all be present, only their order and adjacency are lost.
assertThat("\"quoted phrase\"".escapeForTantivy()).isEqualTo("+\\\"quoted +phrase\\\"")
}
@Test
fun `an unclosed quote no longer fails the search`() {
// Why quotes are escaped at all: an odd number of quotes is a syntax error today, and
// half-typed quotes happen constantly on a mobile keyboard.
assertThat("\"unclosed phrase".escapeForTantivy()).isEqualTo("+\\\"unclosed +phrase")
}
@Test
fun `parentheses are escaped`() {
// An unbalanced paren is a hard parse failure, and parens are common in ordinary prose.
assertThat("(a b)".escapeForTantivy()).isEqualTo("+\\(a +b\\)")
}
@Test
fun `a backslash the user typed is doubled`() {
// Guards the escaper against itself: without this the user's backslash would escape the one
// we appended and shift the whole token. This is why escaping is a single forward pass.
assertThat("foo\\bar".escapeForTantivy()).isEqualTo("+foo\\\\bar")
}
@Test
fun `a plus is left alone when it is not leading`() {
// `+` is only an operator in leading position. Confirms the escaper is not over-broad.
assertThat("C++".escapeForTantivy()).isEqualTo("+C++")
}
@Test
fun `a date with slashes is left alone`() {
// Slashes are only escaped when leading, where they would open a regex literal.
assertThat("1/2/2024".escapeForTantivy()).isEqualTo("+1/2/2024")
}
@Test
fun `a bare-word boolean operator is neutralised`() {
// A uniformity choice rather than purely an error fix: the query is literal text, so
// `a AND b` searches for two words instead of silently switching to boolean AND — which is
// now what every query does anyway.
assertThat("hello NOT".escapeForTantivy()).isEqualTo("+hello +\\NOT")
}
@Test
fun `a token with no letters or digits is never made mandatory`() {
// tantivy's default tokenizer splits on non-alphanumerics, so `:)` and emoji tokenise to
// nothing. As a Must clause a token that indexes no terms can only ever subtract, so an
// emoji typed alongside real words would silently empty the result set. Left as a Should
// clause it is inert, which is what it was before this change too.
assertThat("photo :)".escapeForTantivy()).isEqualTo("+photo \\:\\)")
assertThat("fotoğraf 📷".escapeForTantivy()).isEqualTo("+fotoğraf 📷")
}
@Test
fun `a punctuation-only query behaves exactly as it did before`() {
// The residual cost, recorded honestly: this no longer errors, but it tokenises to nothing
// and becomes EmptyQuery — zero results, silently. Better than killing the search, not good.
assertThat(":::".escapeForTantivy()).isEqualTo("\\:\\:\\:")
}
@Test
fun `a lone star matches nothing instead of everything`() {
// A real behaviour delta: `*` alone is AllQuery today and EmptyQuery after this change.
// Neither is a sensible search, but the change should be pinned rather than discovered.
assertThat("*".escapeForTantivy()).isEqualTo("\\*")
}
@Test
fun `non-ascii words are mandatory like any other`() {
// The account this was built for is Turkish. Kotlin's isLetterOrDigit is Unicode-aware, so
// no accented or non-Latin word is mistaken for punctuation and demoted.
assertThat("fotoğrafa bak".escapeForTantivy()).isEqualTo("+fotoğrafa +bak")
}
@Test
fun `runs of whitespace collapse and the query is trimmed`() {
// Semantically neutral to tantivy — whitespace is only a separator — but it is an
// observable change to the string we hand over, so it is pinned.
assertThat(" spaced out ".escapeForTantivy()).isEqualTo("+spaced +out")
}
@Test
fun `an empty query stays empty`() {
assertThat("".escapeForTantivy()).isEqualTo("")
assertThat(" ".escapeForTantivy()).isEqualTo("")
}
}