From 6509fe63ffb44d832b0a0470d21c3c6a8c023437 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 22 Apr 2025 14:10:20 +0200 Subject: [PATCH 01/13] WidgetDriver: Introduce `to-device` filter. A new widget filter is required to add support for to-device events. This allows to let the widget only send and receive to-device events it has negotiated capabilities for. --- crates/matrix-sdk/src/widget/capabilities.rs | 5 +- crates/matrix-sdk/src/widget/filter.rs | 79 ++++++++++++++++++-- crates/matrix-sdk/src/widget/machine/mod.rs | 6 +- 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/crates/matrix-sdk/src/widget/capabilities.rs b/crates/matrix-sdk/src/widget/capabilities.rs index 9c23ff6405b..1bfe24cefa4 100644 --- a/crates/matrix-sdk/src/widget/capabilities.rs +++ b/crates/matrix-sdk/src/widget/capabilities.rs @@ -60,7 +60,7 @@ pub struct Capabilities { impl Capabilities { /// Checks if a given event is allowed to be forwarded to the widget. /// - /// - `event_filter_input` is a minimized event respresntation that contains + /// - `event_filter_input` is a minimized event representation that contains /// only the information needed to check if the widget is allowed to /// receive the event. (See [`FilterInput`]) pub(super) fn allow_reading<'a>( @@ -78,7 +78,7 @@ impl Capabilities { /// Checks if a given event is allowed to be sent by the widget. /// - /// - `event_filter_input` is a minimized event respresntation that contains + /// - `event_filter_input` is a minimized event representation that contains /// only the information needed to check if the widget is allowed to send /// the event to a matrix room. (See [`FilterInput`]) pub(super) fn allow_sending<'a>( @@ -121,6 +121,7 @@ impl Serialize for Capabilities { match self.0 { Filter::MessageLike(filter) => PrintMessageLikeEventFilter(filter).fmt(f), Filter::State(filter) => PrintStateEventFilter(filter).fmt(f), + Filter::ToDevice(filter) => filter.fmt(f), } } } diff --git a/crates/matrix-sdk/src/widget/filter.rs b/crates/matrix-sdk/src/widget/filter.rs index c63185c16a1..b3f7da20c4c 100644 --- a/crates/matrix-sdk/src/widget/filter.rs +++ b/crates/matrix-sdk/src/widget/filter.rs @@ -12,8 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::fmt; + use ruma::{ - events::{AnyTimelineEvent, MessageLikeEventType, StateEventType}, + events::{ + AnyTimelineEvent, AnyToDeviceEvent, MessageLikeEventType, StateEventType, ToDeviceEventType, + }, serde::Raw, }; use serde::Deserialize; @@ -21,9 +25,9 @@ use tracing::debug; use super::machine::SendEventRequest; -/// A Filter for Matrix events. That is used to decide if a given event can be -/// sent to the widget and if a widgets is allowed to send an event to to a -/// Matrix room or not. +/// A Filter for Matrix events. It is used to decide if a given event can be +/// sent to the widget and if a widget is allowed to send an event to a +/// Matrix room. #[derive(Clone, Debug)] #[cfg_attr(test, derive(PartialEq))] pub enum Filter { @@ -31,6 +35,8 @@ pub enum Filter { MessageLike(MessageLikeEventFilter), /// Filter for state events. State(StateEventFilter), + /// Filter for to device events. + ToDevice(ToDeviceEventFilter), } impl Filter { @@ -41,6 +47,7 @@ impl Filter { match self { Self::MessageLike(filter) => filter.matches(filter_input), Self::State(filter) => filter.matches(filter_input), + Self::ToDevice(filter) => filter.matches(filter_input), } } /// Returns the event type that this filter is configured to match. @@ -51,6 +58,7 @@ impl Filter { match self { Self::MessageLike(filter) => filter.filter_event_type(), Self::State(filter) => filter.filter_event_type(), + Self::ToDevice(filter) => filter.event_type.to_string(), } } } @@ -123,6 +131,33 @@ impl<'a> StateEventFilter { } } +/// Filter for to-device events. +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(PartialEq))] +pub struct ToDeviceEventFilter { + /// The event type this to-device-filter filters for. + pub event_type: ToDeviceEventType, +} + +impl ToDeviceEventFilter { + /// Create a new `ToDeviceEventFilter` with the given event type. + pub fn new(event_type: ToDeviceEventType) -> Self { + Self { event_type } + } +} + +impl ToDeviceEventFilter { + fn matches(&self, filter_input: &FilterInput) -> bool { + matches!(filter_input,FilterInput::ToDevice(f_in) if f_in.event_type == self.event_type) + } +} + +impl fmt::Display for ToDeviceEventFilter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.event_type) + } +} + // Filter input: /// The input data for the filter. This can either be constructed from a @@ -131,8 +166,13 @@ impl<'a> StateEventFilter { #[serde(untagged)] pub enum FilterInput<'a> { #[serde(borrow)] + // The order is important. + // We first need to check if we can deserialize as a state (state_key exists) State(FilterInputState<'a>), + // only then we can check if we can deserialize as a message like. MessageLike(FilterInputMessageLike<'a>), + // ToDevice will need to be done explicitly since it looks the same as a message like. + ToDevice(FilterInputToDevice<'a>), } impl<'a> FilterInput<'a> { @@ -188,10 +228,29 @@ impl<'a> TryFrom<&'a Raw> for FilterInput<'a> { type Error = serde_json::Error; fn try_from(raw_event: &'a Raw) -> Result { + // FilterInput first checks if it can deserialize as a state event (state_key exists) + // and then as a message like event. raw_event.deserialize_as() } } +#[derive(Debug, Deserialize)] +pub struct FilterInputToDevice<'a> { + #[serde(rename = "type")] + pub(super) event_type: &'a str, +} + +/// Create a filter input of type [`FilterInput::ToDevice`]`. +impl<'a> TryFrom<&'a Raw> for FilterInput<'a> { + type Error = serde_json::Error; + fn try_from(raw_event: Raw) -> Result { + // deserialize_as:: will first try state, message like and then to-device. + // The `AnyToDeviceEvent` would match message like first, so we need to explicitly + // deserialize as `FilterInputToDevice`. + raw_event.deserialize_as::().map(FilterInput::ToDevice) + } +} + impl<'a> From<&'a SendEventRequest> for FilterInput<'a> { fn from(request: &'a SendEventRequest) -> Self { match &request.state_key { @@ -234,7 +293,9 @@ mod tests { use super::{ Filter, FilterInput, FilterInputMessageLike, MessageLikeEventFilter, StateEventFilter, }; - use crate::widget::filter::MessageLikeFilterEventContent; + use crate::widget::filter::{ + FilterInputToDevice, MessageLikeFilterEventContent, ToDeviceEventFilter, + }; fn message_event(event_type: &str) -> FilterInput<'_> { FilterInput::MessageLike(FilterInputMessageLike { event_type, content: Default::default() }) @@ -443,4 +504,12 @@ mod tests { assert_eq!(state.state_key, "@alice:example.com"); } } + + #[test] + fn test_to_device_filter_does_match() { + let f = Filter::ToDevice(ToDeviceEventFilter::new("my.custom.to.device".into())); + assert!(f.matches(&FilterInput::ToDevice(FilterInputToDevice { + event_type: "my.custom.to.device".into(), + }))); + } } diff --git a/crates/matrix-sdk/src/widget/machine/mod.rs b/crates/matrix-sdk/src/widget/machine/mod.rs index 1b131254285..282ea25cd0f 100644 --- a/crates/matrix-sdk/src/widget/machine/mod.rs +++ b/crates/matrix-sdk/src/widget/machine/mod.rs @@ -164,16 +164,16 @@ impl WidgetMachine { IncomingMessage::MatrixDriverResponse { request_id, response } => { self.process_matrix_driver_response(request_id, response) } - IncomingMessage::MatrixEventReceived(event) => { + IncomingMessage::MatrixEventReceived(event_raw) => { let CapabilitiesState::Negotiated(capabilities) = &self.capabilities else { error!("Received matrix event before capabilities negotiation"); return Vec::new(); }; capabilities - .allow_reading(&event) + .allow_reading(&event_raw) .then(|| { - self.send_to_widget_request(NotifyNewMatrixEvent(event)) + self.send_to_widget_request(NotifyNewMatrixEvent(event_raw)) .map(|(_request, action)| vec![action]) .unwrap_or_default() }) From 0338b6880189004a4da3bcae91a6fd634477a38c Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 7 Apr 2025 18:52:00 +0200 Subject: [PATCH 02/13] WidgetDriver: add toDevice capability parsing --- crates/matrix-sdk/src/widget/capabilities.rs | 40 ++++++++++++++++--- .../src/widget/machine/tests/capabilities.rs | 13 +++--- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/crates/matrix-sdk/src/widget/capabilities.rs b/crates/matrix-sdk/src/widget/capabilities.rs index 1bfe24cefa4..1f86ce2ccb8 100644 --- a/crates/matrix-sdk/src/widget/capabilities.rs +++ b/crates/matrix-sdk/src/widget/capabilities.rs @@ -22,7 +22,7 @@ use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer} use tracing::{debug, warn}; use super::{ - filter::{Filter, FilterInput}, + filter::{Filter, FilterInput, ToDeviceEventFilter}, MessageLikeEventFilter, StateEventFilter, }; @@ -102,11 +102,13 @@ impl Capabilities { } } -const SEND_EVENT: &str = "org.matrix.msc2762.send.event"; -const READ_EVENT: &str = "org.matrix.msc2762.receive.event"; -const SEND_STATE: &str = "org.matrix.msc2762.send.state_event"; -const READ_STATE: &str = "org.matrix.msc2762.receive.state_event"; -const REQUIRES_CLIENT: &str = "io.element.requires_client"; +pub(super) const SEND_EVENT: &str = "org.matrix.msc2762.send.event"; +pub(super) const READ_EVENT: &str = "org.matrix.msc2762.receive.event"; +pub(super) const SEND_STATE: &str = "org.matrix.msc2762.send.state_event"; +pub(super) const READ_STATE: &str = "org.matrix.msc2762.receive.state_event"; +pub(super) const SEND_TODEVICE: &str = "org.matrix.msc3819.send.to_device"; +pub(super) const READ_TODEVICE: &str = "org.matrix.msc3819.receive.to_device"; +pub(super) const REQUIRES_CLIENT: &str = "io.element.requires_client"; pub(super) const SEND_DELAYED_EVENT: &str = "org.matrix.msc4157.send.delayed_event"; pub(super) const UPDATE_DELAYED_EVENT: &str = "org.matrix.msc4157.update_delayed_event"; @@ -169,6 +171,7 @@ impl Serialize for Capabilities { let name = match filter { Filter::MessageLike(_) => READ_EVENT, Filter::State(_) => READ_STATE, + Filter::ToDevice(_) => READ_TODEVICE, }; seq.serialize_element(&format!("{name}:{}", PrintEventFilter(filter)))?; } @@ -176,6 +179,7 @@ impl Serialize for Capabilities { let name = match filter { Filter::MessageLike(_) => SEND_EVENT, Filter::State(_) => SEND_STATE, + Filter::ToDevice(_) => SEND_TODEVICE, }; seq.serialize_element(&format!("{name}:{}", PrintEventFilter(filter)))?; } @@ -227,6 +231,12 @@ impl<'de> Deserialize<'de> for Capabilities { Some((SEND_STATE, filter_s)) => { Ok(Permission::Send(Filter::State(parse_state_event_filter(filter_s)))) } + Some((READ_TODEVICE, filter_s)) => Ok(Permission::Read(Filter::ToDevice( + parse_to_device_event_filter(filter_s), + ))), + Some((SEND_TODEVICE, filter_s)) => Ok(Permission::Send(Filter::ToDevice( + parse_to_device_event_filter(filter_s), + ))), _ => { debug!("Unknown capability `{s}`"); Ok(Self::Unknown) @@ -253,6 +263,10 @@ impl<'de> Deserialize<'de> for Capabilities { } } + fn parse_to_device_event_filter(s: &str) -> ToDeviceEventFilter { + ToDeviceEventFilter::new(s.into()) + } + let mut capabilities = Capabilities::default(); for capability in Vec::::deserialize(deserializer)? { match capability { @@ -274,6 +288,8 @@ impl<'de> Deserialize<'de> for Capabilities { mod tests { use ruma::events::StateEventType; + use crate::widget::filter::ToDeviceEventFilter; + use super::*; #[test] @@ -294,8 +310,10 @@ mod tests { "org.matrix.msc2762.receive.event:org.matrix.rageshake_request", "org.matrix.msc2762.receive.state_event:m.room.member", "org.matrix.msc2762.receive.state_event:org.matrix.msc3401.call.member", + "org.matrix.msc3819.receive.to_device:io.element.call.encryption_keys", "org.matrix.msc2762.send.event:org.matrix.rageshake_request", "org.matrix.msc2762.send.state_event:org.matrix.msc3401.call.member#@user:matrix.server", + "org.matrix.msc3819.send.to_device:io.element.call.encryption_keys", "org.matrix.msc4157.send.delayed_event", "org.matrix.msc4157.update_delayed_event" ]"#; @@ -308,6 +326,9 @@ mod tests { )), Filter::State(StateEventFilter::WithType(StateEventType::RoomMember)), Filter::State(StateEventFilter::WithType("org.matrix.msc3401.call.member".into())), + Filter::ToDevice(ToDeviceEventFilter::new( + "io.element.call.encryption_keys".into(), + )), ], send: vec![ Filter::MessageLike(MessageLikeEventFilter::WithType( @@ -317,6 +338,9 @@ mod tests { "org.matrix.msc3401.call.member".into(), "@user:matrix.server".into(), )), + Filter::ToDevice(ToDeviceEventFilter::new( + "io.element.call.encryption_keys".into(), + )), ], requires_client: true, update_delayed_event: true, @@ -336,6 +360,9 @@ mod tests { "org.matrix.msc3401.call.member".into(), "@user:matrix.server".into(), )), + Filter::ToDevice(ToDeviceEventFilter::new( + "io.element.call.encryption_keys".into(), + )), ], send: vec![ Filter::MessageLike(MessageLikeEventFilter::WithType("io.element.custom".into())), @@ -343,6 +370,7 @@ mod tests { "org.matrix.msc3401.call.member".into(), "@user:matrix.server".into(), )), + Filter::ToDevice(ToDeviceEventFilter::new("my.org.other.to_device_event".into())), ], requires_client: true, update_delayed_event: false, diff --git a/crates/matrix-sdk/src/widget/machine/tests/capabilities.rs b/crates/matrix-sdk/src/widget/machine/tests/capabilities.rs index 0b221a3a17f..8ac1f591448 100644 --- a/crates/matrix-sdk/src/widget/machine/tests/capabilities.rs +++ b/crates/matrix-sdk/src/widget/machine/tests/capabilities.rs @@ -18,8 +18,12 @@ use ruma::owned_room_id; use serde_json::{from_value, json}; use super::{parse_msg, WIDGET_ID}; -use crate::widget::machine::{ - incoming::MatrixDriverResponse, Action, IncomingMessage, MatrixDriverRequestData, WidgetMachine, +use crate::widget::{ + capabilities::{READ_EVENT, READ_STATE, READ_TODEVICE}, + machine::{ + incoming::MatrixDriverResponse, Action, IncomingMessage, MatrixDriverRequestData, + WidgetMachine, + }, }; #[test] @@ -191,10 +195,7 @@ pub(super) fn assert_capabilities_dance( }; // We get the `Subscribe` command if we requested some reading capabilities. - if ["org.matrix.msc2762.receive.state_event", "org.matrix.msc2762.receive.event"] - .into_iter() - .any(|c| capability.starts_with(c)) - { + if [READ_EVENT, READ_STATE, READ_TODEVICE].into_iter().any(|c| capability.starts_with(c)) { let action = actions.remove(0); assert_matches!(action, Action::Subscribe); } From 023822e48e9847de9725ec0a66be03a517b86117 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 22 Apr 2025 15:51:31 +0200 Subject: [PATCH 03/13] WidgetDriver: ffi for toDevice filter --- bindings/matrix-sdk-ffi/src/widget.rs | 10 +++++++++- crates/matrix-sdk/src/widget/mod.rs | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bindings/matrix-sdk-ffi/src/widget.rs b/bindings/matrix-sdk-ffi/src/widget.rs index ff2a77e93d6..e918df6e293 100644 --- a/bindings/matrix-sdk-ffi/src/widget.rs +++ b/bindings/matrix-sdk-ffi/src/widget.rs @@ -4,7 +4,7 @@ use async_compat::get_runtime_handle; use language_tags::LanguageTag; use matrix_sdk::{ async_trait, - widget::{MessageLikeEventFilter, StateEventFilter}, + widget::{MessageLikeEventFilter, StateEventFilter, ToDeviceEventFilter}, }; use ruma::events::MessageLikeEventType; use tracing::error; @@ -488,6 +488,8 @@ pub enum WidgetEventFilter { StateWithType { event_type: String }, /// Matches state events with the given `type` and `state_key`. StateWithTypeAndStateKey { event_type: String, state_key: String }, + /// Matches to-device events with the given `event_type`. + ToDevice { event_type: String }, } impl From for matrix_sdk::widget::Filter { @@ -505,6 +507,9 @@ impl From for matrix_sdk::widget::Filter { WidgetEventFilter::StateWithTypeAndStateKey { event_type, state_key } => { Self::State(StateEventFilter::WithTypeAndStateKey(event_type.into(), state_key)) } + WidgetEventFilter::ToDevice { event_type } => { + Self::ToDevice(ToDeviceEventFilter { event_type: event_type.into() }) + } } } } @@ -526,6 +531,9 @@ impl From for WidgetEventFilter { F::State(StateEventFilter::WithTypeAndStateKey(event_type, state_key)) => { Self::StateWithTypeAndStateKey { event_type: event_type.to_string(), state_key } } + F::ToDevice(ToDeviceEventFilter { event_type }) => { + Self::ToDevice { event_type: event_type.to_string() } + } } } } diff --git a/crates/matrix-sdk/src/widget/mod.rs b/crates/matrix-sdk/src/widget/mod.rs index a8a4f24e31d..211e5b29d52 100644 --- a/crates/matrix-sdk/src/widget/mod.rs +++ b/crates/matrix-sdk/src/widget/mod.rs @@ -40,7 +40,7 @@ mod settings; pub use self::{ capabilities::{Capabilities, CapabilitiesProvider}, - filter::{Filter, MessageLikeEventFilter, StateEventFilter}, + filter::{Filter, MessageLikeEventFilter, StateEventFilter, ToDeviceEventFilter}, settings::{ ClientProperties, EncryptionSystem, Intent, VirtualElementCallWidgetOptions, WidgetSettings, }, From 9f84d9ad23d614ccba729651193a1a9acb72f11b Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 7 Apr 2025 18:51:22 +0200 Subject: [PATCH 04/13] WidgetDriver: add ToDevice widget events and machine actions. It consists of the following changes: - add a `NotifyNewToDeviceEvent` ToWidget request (a request that will be sent to the widget from the client when the client receives a widget action over the widget api) - add the `SendToDeviceRequest` (driver request that will be sent from the widget and asks the driver to send a ToDevice event) - add the ToDeviceActions to the required enums: `IncomingMessage`(machine), `MatrixDriverResponse`, `FromWidgetResponse`, `FromWidgetRequest`, `MatrixDriverRequestData` --- crates/matrix-sdk/src/widget/filter.rs | 16 +++-- .../src/widget/machine/driver_req.rs | 53 ++++++++++++++++- .../src/widget/machine/from_widget.rs | 23 +++++++- .../matrix-sdk/src/widget/machine/incoming.rs | 11 +++- crates/matrix-sdk/src/widget/machine/mod.rs | 59 +++++++++++++++++-- .../src/widget/machine/to_widget.rs | 16 ++++- crates/matrix-sdk/src/widget/mod.rs | 2 + 7 files changed, 162 insertions(+), 18 deletions(-) diff --git a/crates/matrix-sdk/src/widget/filter.rs b/crates/matrix-sdk/src/widget/filter.rs index b3f7da20c4c..72fdf384740 100644 --- a/crates/matrix-sdk/src/widget/filter.rs +++ b/crates/matrix-sdk/src/widget/filter.rs @@ -23,7 +23,7 @@ use ruma::{ use serde::Deserialize; use tracing::debug; -use super::machine::SendEventRequest; +use super::machine::{SendEventRequest, SendToDeviceRequest}; /// A Filter for Matrix events. It is used to decide if a given event can be /// sent to the widget and if a widget is allowed to send an event to a @@ -147,8 +147,8 @@ impl ToDeviceEventFilter { } impl ToDeviceEventFilter { - fn matches(&self, filter_input: &FilterInput) -> bool { - matches!(filter_input,FilterInput::ToDevice(f_in) if f_in.event_type == self.event_type) + fn matches(&self, filter_input: &FilterInput<'_>) -> bool { + matches!(filter_input,FilterInput::ToDevice(f_in) if f_in.event_type == self.event_type.to_string()) } } @@ -243,11 +243,17 @@ pub struct FilterInputToDevice<'a> { /// Create a filter input of type [`FilterInput::ToDevice`]`. impl<'a> TryFrom<&'a Raw> for FilterInput<'a> { type Error = serde_json::Error; - fn try_from(raw_event: Raw) -> Result { + fn try_from(raw_event: &'a Raw) -> Result { // deserialize_as:: will first try state, message like and then to-device. // The `AnyToDeviceEvent` would match message like first, so we need to explicitly // deserialize as `FilterInputToDevice`. - raw_event.deserialize_as::().map(FilterInput::ToDevice) + raw_event.deserialize_as::>().map(FilterInput::ToDevice) + } +} + +impl<'a> From<&'a SendToDeviceRequest> for FilterInput<'a> { + fn from(request: &'a SendToDeviceRequest) -> Self { + FilterInput::ToDevice(FilterInputToDevice { event_type: &request.event_type }) } } diff --git a/crates/matrix-sdk/src/widget/machine/driver_req.rs b/crates/matrix-sdk/src/widget/machine/driver_req.rs index d2808439ff1..3c2a995955d 100644 --- a/crates/matrix-sdk/src/widget/machine/driver_req.rs +++ b/crates/matrix-sdk/src/widget/machine/driver_req.rs @@ -14,12 +14,17 @@ //! A high-level API for requests that we send to the matrix driver. -use std::marker::PhantomData; +use std::{collections::BTreeMap, marker::PhantomData}; use ruma::{ - api::client::{account::request_openid_token, delayed_events::update_delayed_event}, - events::AnyTimelineEvent, + api::client::{ + account::request_openid_token, delayed_events::update_delayed_event, + to_device::send_event_to_device, + }, + events::{AnyTimelineEvent, AnyToDeviceEventContent}, serde::Raw, + to_device::DeviceIdOrAllDevices, + OwnedUserId, }; use serde::Deserialize; use serde_json::value::RawValue as RawJsonValue; @@ -52,6 +57,9 @@ pub(crate) enum MatrixDriverRequestData { /// Send matrix event that corresponds to the given description. SendMatrixEvent(SendEventRequest), + /// Send matrix event that corresponds to the given description. + SendToDeviceEvent(SendToDeviceRequest), + /// Data for sending a UpdateDelayedEvent client server api request. UpdateDelayedEvent(UpdateDelayedEventRequest), } @@ -253,6 +261,45 @@ impl FromMatrixDriverResponse for SendEventResponse { } } +/// Ask the client to send matrix event that corresponds to the given +/// description and returns an event ID (or a delay ID, +/// see [MSC4140](https://github.com/matrix-org/matrix-spec-proposals/pull/4140)) as a response. +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct SendToDeviceRequest { + /// The type of the event. + #[serde(rename = "type")] + pub(crate) event_type: String, + // If the to_device message should be encrypted or not. + pub(crate) encrypted: bool, + /// The messages that will be encrypted (per device) and sent. + /// They are organized in a map of user_id -> device_id -> content like the + /// cs api request. + pub(crate) messages: + BTreeMap>>, +} + +impl From for MatrixDriverRequestData { + fn from(value: SendToDeviceRequest) -> Self { + MatrixDriverRequestData::SendToDeviceEvent(value) + } +} + +impl MatrixDriverRequest for SendToDeviceRequest { + type Response = send_event_to_device::v3::Response; +} + +impl FromMatrixDriverResponse for send_event_to_device::v3::Response { + fn from_response(ev: MatrixDriverResponse) -> Option { + match ev { + MatrixDriverResponse::MatrixToDeviceSent(response) => Some(response), + _ => { + error!("bug in MatrixDriver, received wrong event response"); + None + } + } + } +} + /// Ask the client to send a UpdateDelayedEventRequest with the given `delay_id` /// and `action`. Defined by [MSC4157](https://github.com/matrix-org/matrix-spec-proposals/pull/4157) #[derive(Deserialize, Debug, Clone)] diff --git a/crates/matrix-sdk/src/widget/machine/from_widget.rs b/crates/matrix-sdk/src/widget/machine/from_widget.rs index fa04c3baf54..0f21448cdc5 100644 --- a/crates/matrix-sdk/src/widget/machine/from_widget.rs +++ b/crates/matrix-sdk/src/widget/machine/from_widget.rs @@ -17,6 +17,7 @@ use ruma::{ api::client::{ delayed_events::{delayed_message_event, delayed_state_event, update_delayed_event}, error::{ErrorBody, StandardErrorBody}, + to_device::send_event_to_device, }, events::AnyTimelineEvent, serde::Raw, @@ -24,7 +25,7 @@ use ruma::{ }; use serde::{Deserialize, Serialize}; -use super::{SendEventRequest, UpdateDelayedEventRequest}; +use super::{driver_req::SendToDeviceRequest, SendEventRequest, UpdateDelayedEventRequest}; use crate::{widget::StateKeySelector, Error, HttpError, RumaApiError}; #[derive(Deserialize, Debug)] @@ -37,6 +38,7 @@ pub(super) enum FromWidgetRequest { #[serde(rename = "org.matrix.msc2876.read_events")] ReadEvent(ReadEventRequest), SendEvent(SendEventRequest), + SendToDevice(SendToDeviceRequest), #[serde(rename = "org.matrix.msc4157.update_delayed_event")] DelayedEventUpdate(UpdateDelayedEventRequest), } @@ -68,7 +70,7 @@ impl FromWidgetErrorResponse { } } - /// Create a error response to send to the widget from a matrix sdk error. + /// Create an error response to send to the widget from a matrix sdk error. pub(crate) fn from_error(error: Error) -> Self { match error { Error::Http(e) => FromWidgetErrorResponse::from_http_error(*e), @@ -97,6 +99,7 @@ struct FromWidgetError { message: String, /// Optional matrix error hinting at workarounds for specific errors. + #[serde(skip_serializing_if = "Option::is_none")] matrix_api_error: Option, } @@ -230,6 +233,8 @@ impl From for SendEventResponse { /// [`update_delayed_event`](update_delayed_event::unstable::Response) /// which derives Serialize. (The response struct from Ruma does not derive /// serialize) +/// This is intentionally an empty tuple struct (not a unit struct), so that it +/// serializes to `{}` instead of `Null` when returned to the widget as json. #[derive(Serialize, Debug)] pub(crate) struct UpdateDelayedEventResponse {} impl From for UpdateDelayedEventResponse { @@ -237,3 +242,17 @@ impl From for UpdateDelayedEventRespon Self {} } } + +/// The response to the widget that it received the to-device event. +/// Only used as the response for the successful send case. +/// FromWidgetErrorResponse will be used otherwise. +/// This is intentionally an empty tuple struct (not a unit struct), so that it +/// serializes to `{}` instead of `Null` when returned to the widget as json. +#[derive(Serialize, Debug)] +pub(crate) struct SendToDeviceEventResponse {} + +impl From for SendToDeviceEventResponse { + fn from(_: send_event_to_device::v3::Response) -> Self { + Self {} + } +} diff --git a/crates/matrix-sdk/src/widget/machine/incoming.rs b/crates/matrix-sdk/src/widget/machine/incoming.rs index d90f3740869..1edb23fb581 100644 --- a/crates/matrix-sdk/src/widget/machine/incoming.rs +++ b/crates/matrix-sdk/src/widget/machine/incoming.rs @@ -13,8 +13,8 @@ // limitations under the License. use ruma::{ - api::client::{account::request_openid_token, delayed_events}, - events::AnyTimelineEvent, + api::client::{account::request_openid_token, delayed_events, to_device::send_event_to_device}, + events::{AnyTimelineEvent, AnyToDeviceEvent}, serde::Raw, }; use serde::{de, Deserialize, Deserializer}; @@ -47,8 +47,12 @@ pub(crate) enum IncomingMessage { /// The `MatrixDriver` notified the `WidgetMachine` of a new matrix event. /// /// This means that the machine previously subscribed to some events - /// ([`crate::widget::Action::Subscribe`] request). + /// ([`crate::widget::Action::SubscribeTimeline`] request). MatrixEventReceived(Raw), + + /// The `MatrixDriver` notified the `WidgetMachine` of a new to_device + /// event. + ToDeviceReceived(Raw), } pub(crate) enum MatrixDriverResponse { @@ -65,6 +69,7 @@ pub(crate) enum MatrixDriverResponse { /// Client sent some matrix event. The response contains the event ID. /// A response to an `Action::SendMatrixEvent` command. MatrixEventSent(SendEventResponse), + MatrixToDeviceSent(send_event_to_device::v3::Response), MatrixDelayedEventUpdate(delayed_events::update_delayed_event::unstable::Response), } diff --git a/crates/matrix-sdk/src/widget/machine/mod.rs b/crates/matrix-sdk/src/widget/machine/mod.rs index 282ea25cd0f..aeb89657ddb 100644 --- a/crates/matrix-sdk/src/widget/machine/mod.rs +++ b/crates/matrix-sdk/src/widget/machine/mod.rs @@ -17,7 +17,7 @@ use std::time::Duration; use driver_req::UpdateDelayedEventRequest; -use from_widget::UpdateDelayedEventResponse; +use from_widget::{SendToDeviceEventResponse, UpdateDelayedEventResponse}; use indexmap::IndexMap; use ruma::{ serde::{JsonObject, Raw}, @@ -41,8 +41,9 @@ use self::{ openid::{OpenIdResponse, OpenIdState}, pending::{PendingRequests, RequestLimits}, to_widget::{ - NotifyCapabilitiesChanged, NotifyNewMatrixEvent, NotifyOpenIdChanged, RequestCapabilities, - ToWidgetRequest, ToWidgetRequestHandle, ToWidgetResponse, + NotifyCapabilitiesChanged, NotifyNewMatrixEvent, NotifyNewToDeviceEvent, + NotifyOpenIdChanged, RequestCapabilities, ToWidgetRequest, ToWidgetRequestHandle, + ToWidgetResponse, }, }; #[cfg(doc)] @@ -64,7 +65,9 @@ mod tests; mod to_widget; pub(crate) use self::{ - driver_req::{MatrixDriverRequestData, ReadStateEventRequest, SendEventRequest}, + driver_req::{ + MatrixDriverRequestData, ReadStateEventRequest, SendEventRequest, SendToDeviceRequest, + }, from_widget::SendEventResponse, incoming::{IncomingMessage, MatrixDriverResponse}, }; @@ -179,6 +182,21 @@ impl WidgetMachine { }) .unwrap_or_default() } + IncomingMessage::ToDeviceReceived(to_device_raw) => { + let CapabilitiesState::Negotiated(capabilities) = &self.capabilities else { + error!("Received to device event before capabilities negotiation"); + return Vec::new(); + }; + + capabilities + .allow_reading(&to_device_raw) + .then(|| { + self.send_to_widget_request(NotifyNewToDeviceEvent(to_device_raw)) + .map(|(_request, action)| vec![action]) + .unwrap_or_default() + }) + .unwrap_or_default() + } } } @@ -247,6 +265,11 @@ impl WidgetMachine { .map(|a| vec![a]) .unwrap_or_default(), + FromWidgetRequest::SendToDevice(req) => self + .process_to_device_request(req, raw_request) + .map(|a| vec![a]) + .unwrap_or_default(), + FromWidgetRequest::GetOpenId {} => { let mut actions = vec![Self::send_from_widget_response(raw_request, Ok(OpenIdResponse::Pending))]; @@ -436,7 +459,35 @@ impl WidgetMachine { result.map_err(FromWidgetErrorResponse::from_error), )] }); + Some(action) + } + fn process_to_device_request( + &mut self, + request: SendToDeviceRequest, + raw_request: Raw, + ) -> Option { + let CapabilitiesState::Negotiated(capabilities) = &self.capabilities else { + error!("Received send event request before capabilities negotiation"); + return None; + }; + + if !capabilities.allow_sending(&request) { + return Some(Self::send_from_widget_error_string_response( + raw_request, + format!("Not allowed to send to-device message of type: {}", request.event_type), + )); + } + + let (request, action) = self.send_matrix_driver_request(request)?; + request.then(|result, _| { + vec![Self::send_from_widget_response( + raw_request, + result + .map(Into::::into) + .map_err(FromWidgetErrorResponse::from_error), + )] + }); Some(action) } diff --git a/crates/matrix-sdk/src/widget/machine/to_widget.rs b/crates/matrix-sdk/src/widget/machine/to_widget.rs index 45e6498fbef..0d68fbbafe3 100644 --- a/crates/matrix-sdk/src/widget/machine/to_widget.rs +++ b/crates/matrix-sdk/src/widget/machine/to_widget.rs @@ -14,7 +14,10 @@ use std::marker::PhantomData; -use ruma::{events::AnyTimelineEvent, serde::Raw}; +use ruma::{ + events::{AnyTimelineEvent, AnyToDeviceEvent}, + serde::Raw, +}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::value::RawValue as RawJsonValue; use tracing::error; @@ -124,3 +127,14 @@ impl ToWidgetRequest for NotifyNewMatrixEvent { #[derive(Deserialize)] pub(crate) struct Empty {} + +/// Notify the widget that we received a new matrix event. +/// This is a "response" to the widget subscribing to the events in the room. +#[derive(Serialize)] +#[serde(transparent)] +pub(crate) struct NotifyNewToDeviceEvent(pub(crate) Raw); + +impl ToWidgetRequest for NotifyNewToDeviceEvent { + const ACTION: &'static str = "send_to_device"; + type ResponseData = Empty; +} diff --git a/crates/matrix-sdk/src/widget/mod.rs b/crates/matrix-sdk/src/widget/mod.rs index 211e5b29d52..e1d72287898 100644 --- a/crates/matrix-sdk/src/widget/mod.rs +++ b/crates/matrix-sdk/src/widget/mod.rs @@ -236,6 +236,8 @@ impl WidgetDriver { .update_delayed_event(req.delay_id, req.action) .await .map(MatrixDriverResponse::MatrixDelayedEventUpdate), + + MatrixDriverRequestData::SendToDeviceEvent(req) => todo!(), }; // Forward the matrix driver response to the incoming message stream. From b643b685cfa669922b984eb027b654bd3f3c4475 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 23 Apr 2025 13:18:02 +0200 Subject: [PATCH 05/13] WidgetDriver: add matrix driver toDevice support (reading and sending events via cs api) This also hooks up the widget via the machine actions. And adds toDevice events to the subscription. --- .../src/widget/machine/to_widget.rs | 2 +- crates/matrix-sdk/src/widget/matrix.rs | 137 +++++++++++++++--- crates/matrix-sdk/src/widget/mod.rs | 21 ++- 3 files changed, 137 insertions(+), 23 deletions(-) diff --git a/crates/matrix-sdk/src/widget/machine/to_widget.rs b/crates/matrix-sdk/src/widget/machine/to_widget.rs index 0d68fbbafe3..b89961b253e 100644 --- a/crates/matrix-sdk/src/widget/machine/to_widget.rs +++ b/crates/matrix-sdk/src/widget/machine/to_widget.rs @@ -128,7 +128,7 @@ impl ToWidgetRequest for NotifyNewMatrixEvent { #[derive(Deserialize)] pub(crate) struct Empty {} -/// Notify the widget that we received a new matrix event. +/// Notify the widget that we received a new matrix to device event. /// This is a "response" to the widget subscribing to the events in the room. #[derive(Serialize)] #[serde(transparent)] diff --git a/crates/matrix-sdk/src/widget/matrix.rs b/crates/matrix-sdk/src/widget/matrix.rs index 0f6ccec06a1..cab522341cf 100644 --- a/crates/matrix-sdk/src/widget/matrix.rs +++ b/crates/matrix-sdk/src/widget/matrix.rs @@ -17,21 +17,24 @@ use std::collections::BTreeMap; -use matrix_sdk_base::deserialized_responses::RawAnySyncOrStrippedState; +use matrix_sdk_base::deserialized_responses::{EncryptionInfo, RawAnySyncOrStrippedState}; use ruma::{ api::client::{ account::request_openid_token::v3::{Request as OpenIdRequest, Response as OpenIdResponse}, delayed_events::{self, update_delayed_event::unstable::UpdateAction}, filter::RoomEventFilter, + to_device::send_event_to_device::{self, v3::Request as RumaToDeviceRequest}, }, assign, events::{ AnyMessageLikeEventContent, AnyStateEventContent, AnySyncMessageLikeEvent, - AnySyncStateEvent, AnySyncTimelineEvent, AnyTimelineEvent, MessageLikeEventType, - StateEventType, TimelineEventType, + AnySyncStateEvent, AnySyncTimelineEvent, AnyTimelineEvent, AnyToDeviceEvent, + AnyToDeviceEventContent, MessageLikeEventType, StateEventType, TimelineEventType, + ToDeviceEventType, }, serde::{from_raw_json_value, Raw}, - EventId, RoomId, TransactionId, + to_device::DeviceIdOrAllDevices, + EventId, OwnedUserId, RoomId, TransactionId, }; use serde_json::{value::RawValue as RawJsonValue, Value}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; @@ -86,7 +89,11 @@ impl MatrixDriver { ) -> Result>> { let room_id = self.room.room_id(); let convert = |sync_or_stripped_state| match sync_or_stripped_state { - RawAnySyncOrStrippedState::Sync(ev) => Some(attach_room_id(ev.cast_ref(), room_id)), + RawAnySyncOrStrippedState::Sync(ev) => with_attached_room_id(ev.cast_ref(), room_id) + .map_err(|e| { + error!("failed to convert event from `get_state_event` response:{}", e) + }) + .ok(), RawAnySyncOrStrippedState::Stripped(_) => { error!("MatrixDriver can't operate in invited rooms"); None @@ -181,7 +188,7 @@ impl MatrixDriver { /// Starts forwarding new room events. Once the returned `EventReceiver` /// is dropped, forwarding will be stopped. - pub(crate) fn events(&self) -> EventReceiver { + pub(crate) fn events(&self) -> EventReceiver> { let (tx, rx) = unbounded_channel(); let room_id = self.room.room_id().to_owned(); @@ -190,14 +197,29 @@ impl MatrixDriver { let _room_id = room_id.clone(); let handle_msg_like = self.room.add_event_handler(move |raw: Raw| { - let _ = _tx.send(attach_room_id(raw.cast_ref(), &_room_id)); + match with_attached_room_id(raw.cast_ref(), &_room_id) { + Ok(event_with_room_id) => { + let _ = _tx.send(event_with_room_id); + } + Err(e) => { + error!("Failed to attach room id to message like event: {}", e); + } + } async {} }); let drop_guard_msg_like = self.room.client().event_handler_drop_guard(handle_msg_like); - + let _room_id = room_id; + let _tx = tx; // Get only all state events from the state section of the sync. let handle_state = self.room.add_event_handler(move |raw: Raw| { - let _ = tx.send(attach_room_id(raw.cast_ref(), &room_id)); + match with_attached_room_id(raw.cast_ref(), &_room_id) { + Ok(event_with_room_id) => { + let _ = _tx.send(event_with_room_id); + } + Err(e) => { + error!("Failed to attach room id to state event: {}", e); + } + } async {} }); let drop_guard_state = self.room.client().event_handler_drop_guard(handle_state); @@ -208,25 +230,102 @@ impl MatrixDriver { // section of the sync will not be forwarded to the widget. // TODO annotate the events and send both timeline and state section state // events. - EventReceiver { rx, _drop_guards: [drop_guard_msg_like, drop_guard_state] } + EventReceiver { rx, _drop_guards: vec![drop_guard_msg_like, drop_guard_state] } + } + + /// Starts forwarding new room events. Once the returned `EventReceiver` + /// is dropped, forwarding will be stopped. + pub(crate) fn to_device_events(&self) -> EventReceiver> { + let (tx, rx) = unbounded_channel(); + + let to_device_handle = self.room.client().add_event_handler( + move |raw: Raw, encryption_info: Option| { + match with_attached_encryption_flag(raw, &encryption_info) { + Ok(ev) => { + let _ = tx.send(ev); + } + Err(e) => { + error!("Failed to attach encryption flag to to_device event: {}", e); + } + } + async {} + }, + ); + + let drop_guard = self.room.client().event_handler_drop_guard(to_device_handle); + EventReceiver { rx, _drop_guards: vec![drop_guard] } + } + + /// It will ignore all devices where errors occurred or where the device is + /// not verified or where th user has a has_verification_violation. + pub(crate) async fn send_to_device( + &self, + event_type: ToDeviceEventType, + encrypted: bool, + messages: BTreeMap< + OwnedUserId, + BTreeMap>, + >, + ) -> Result { + let client = self.room.client(); + + let request = if encrypted { + return Err(Error::UnknownError( + "Sending encrypted to_device events is not supported by the widget driver.".into(), + )); + } else { + RumaToDeviceRequest::new_raw(event_type, TransactionId::new(), messages) + }; + + let response = client.send(request).await; + + response.map_err(Into::into) } } /// A simple entity that wraps an `UnboundedReceiver` /// along with the drop guard for the room event handler. -pub(crate) struct EventReceiver { - rx: UnboundedReceiver>, - _drop_guards: [EventHandlerDropGuard; 2], +pub(crate) struct EventReceiver { + rx: UnboundedReceiver, + _drop_guards: Vec, } -impl EventReceiver { - pub(crate) async fn recv(&mut self) -> Option> { +impl EventReceiver { + pub(crate) async fn recv(&mut self) -> Option { self.rx.recv().await } } -fn attach_room_id(raw_ev: &Raw, room_id: &RoomId) -> Raw { - let mut ev_obj = raw_ev.deserialize_as::>>().unwrap(); - ev_obj.insert("room_id".to_owned(), serde_json::value::to_raw_value(room_id).unwrap()); - Raw::new(&ev_obj).unwrap().cast() +/// Attach a room id to the event. This is needed because the widget API +/// requires the room id to be present in the event. + +fn with_attached_room_id( + raw: &Raw, + room_id: &RoomId, +) -> Result> { + // This is the only modification we need to do to the events otherwise they are + // just forwarded raw to the widget. + // This is why we do the serialization dance here to allow the optimization of + // using `BTreeMap` instead of serializing the full event. + match raw.deserialize_as::>>() { + Ok(mut ev_mut) => { + ev_mut.insert("room_id".to_owned(), serde_json::value::to_raw_value(room_id)?); + Ok(Raw::new(&ev_mut)?.cast()) + } + Err(e) => Err(Error::from(e)), + } +} + +fn with_attached_encryption_flag( + raw: Raw, + encryption_info: &Option, +) -> Result> { + match raw.deserialize_as::>>() { + Ok(mut ev_mut) => { + let encrypted = encryption_info.is_some(); + ev_mut.insert("encrypted".to_owned(), serde_json::value::to_raw_value(&encrypted)?); + Ok(Raw::new(&ev_mut)?.cast()) + } + Err(e) => Err(Error::from(e)), + } } diff --git a/crates/matrix-sdk/src/widget/mod.rs b/crates/matrix-sdk/src/widget/mod.rs index e1d72287898..8568a90d841 100644 --- a/crates/matrix-sdk/src/widget/mod.rs +++ b/crates/matrix-sdk/src/widget/mod.rs @@ -237,7 +237,16 @@ impl WidgetDriver { .await .map(MatrixDriverResponse::MatrixDelayedEventUpdate), - MatrixDriverRequestData::SendToDeviceEvent(req) => todo!(), + MatrixDriverRequestData::SendToDeviceEvent(send_to_device_request) => { + matrix_driver + .send_to_device( + send_to_device_request.event_type.into(), + send_to_device_request.encrypted, + send_to_device_request.messages, + ) + .await + .map(MatrixDriverResponse::MatrixToDeviceSent) + } }; // Forward the matrix driver response to the incoming message stream. @@ -259,7 +268,8 @@ impl WidgetDriver { self.event_forwarding_guard = Some(guard); - let mut matrix = matrix_driver.events(); + let mut events_receiver = matrix_driver.events(); + let mut to_device_receiver = matrix_driver.to_device_events(); let incoming_msg_tx = incoming_msg_tx.clone(); spawn(async move { @@ -270,10 +280,15 @@ impl WidgetDriver { return; } - Some(event) = matrix.recv() => { + Some(event) = events_receiver.recv() => { // Forward all events to the incoming messages stream. let _ = incoming_msg_tx.send(IncomingMessage::MatrixEventReceived(event)); } + + Some(event) = to_device_receiver.recv() => { + // Forward all events to the incoming messages stream. + let _ = incoming_msg_tx.send(IncomingMessage::ToDeviceReceived(event)); + } } } }); From e21116a0aed6f192de02c5a4da5d355b29915445 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 30 Apr 2025 08:51:50 +0200 Subject: [PATCH 06/13] fixup use custom type to serialize with references --- crates/matrix-sdk/src/widget/capabilities.rs | 3 +- crates/matrix-sdk/src/widget/filter.rs | 12 +-- crates/matrix-sdk/src/widget/matrix.rs | 90 +++++++++++--------- 3 files changed, 56 insertions(+), 49 deletions(-) diff --git a/crates/matrix-sdk/src/widget/capabilities.rs b/crates/matrix-sdk/src/widget/capabilities.rs index 1f86ce2ccb8..aae3672b82a 100644 --- a/crates/matrix-sdk/src/widget/capabilities.rs +++ b/crates/matrix-sdk/src/widget/capabilities.rs @@ -288,9 +288,8 @@ impl<'de> Deserialize<'de> for Capabilities { mod tests { use ruma::events::StateEventType; - use crate::widget::filter::ToDeviceEventFilter; - use super::*; + use crate::widget::filter::ToDeviceEventFilter; #[test] fn deserialization_of_no_capabilities() { diff --git a/crates/matrix-sdk/src/widget/filter.rs b/crates/matrix-sdk/src/widget/filter.rs index 72fdf384740..51fcac7c042 100644 --- a/crates/matrix-sdk/src/widget/filter.rs +++ b/crates/matrix-sdk/src/widget/filter.rs @@ -228,8 +228,8 @@ impl<'a> TryFrom<&'a Raw> for FilterInput<'a> { type Error = serde_json::Error; fn try_from(raw_event: &'a Raw) -> Result { - // FilterInput first checks if it can deserialize as a state event (state_key exists) - // and then as a message like event. + // FilterInput first checks if it can deserialize as a state event (state_key + // exists) and then as a message like event. raw_event.deserialize_as() } } @@ -244,9 +244,9 @@ pub struct FilterInputToDevice<'a> { impl<'a> TryFrom<&'a Raw> for FilterInput<'a> { type Error = serde_json::Error; fn try_from(raw_event: &'a Raw) -> Result { - // deserialize_as:: will first try state, message like and then to-device. - // The `AnyToDeviceEvent` would match message like first, so we need to explicitly - // deserialize as `FilterInputToDevice`. + // deserialize_as:: will first try state, message like and then + // to-device. The `AnyToDeviceEvent` would match message like first, so + // we need to explicitly deserialize as `FilterInputToDevice`. raw_event.deserialize_as::>().map(FilterInput::ToDevice) } } @@ -515,7 +515,7 @@ mod tests { fn test_to_device_filter_does_match() { let f = Filter::ToDevice(ToDeviceEventFilter::new("my.custom.to.device".into())); assert!(f.matches(&FilterInput::ToDevice(FilterInputToDevice { - event_type: "my.custom.to.device".into(), + event_type: "my.custom.to.device", }))); } } diff --git a/crates/matrix-sdk/src/widget/matrix.rs b/crates/matrix-sdk/src/widget/matrix.rs index cab522341cf..9bfa1e2adbd 100644 --- a/crates/matrix-sdk/src/widget/matrix.rs +++ b/crates/matrix-sdk/src/widget/matrix.rs @@ -28,14 +28,14 @@ use ruma::{ assign, events::{ AnyMessageLikeEventContent, AnyStateEventContent, AnySyncMessageLikeEvent, - AnySyncStateEvent, AnySyncTimelineEvent, AnyTimelineEvent, AnyToDeviceEvent, - AnyToDeviceEventContent, MessageLikeEventType, StateEventType, TimelineEventType, - ToDeviceEventType, + AnySyncStateEvent, AnyTimelineEvent, AnyToDeviceEvent, AnyToDeviceEventContent, + MessageLikeEventType, StateEventType, TimelineEventType, ToDeviceEventType, }, serde::{from_raw_json_value, Raw}, to_device::DeviceIdOrAllDevices, - EventId, OwnedUserId, RoomId, TransactionId, + EventId, OwnedRoomId, OwnedUserId, TransactionId, }; +use serde::{Deserialize, Serialize}; use serde_json::{value::RawValue as RawJsonValue, Value}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; use tracing::error; @@ -89,11 +89,14 @@ impl MatrixDriver { ) -> Result>> { let room_id = self.room.room_id(); let convert = |sync_or_stripped_state| match sync_or_stripped_state { - RawAnySyncOrStrippedState::Sync(ev) => with_attached_room_id(ev.cast_ref(), room_id) - .map_err(|e| { - error!("failed to convert event from `get_state_event` response:{}", e) - }) - .ok(), + RawAnySyncOrStrippedState::Sync(ev) => { + add_props_to_raw(&ev, Some(room_id.to_owned()), None) + .map(Raw::cast) + .map_err(|e| { + error!("failed to convert event from `get_state_event` response:{}", e) + }) + .ok() + } RawAnySyncOrStrippedState::Stripped(_) => { error!("MatrixDriver can't operate in invited rooms"); None @@ -197,9 +200,9 @@ impl MatrixDriver { let _room_id = room_id.clone(); let handle_msg_like = self.room.add_event_handler(move |raw: Raw| { - match with_attached_room_id(raw.cast_ref(), &_room_id) { + match add_props_to_raw(&raw, Some(_room_id), None) { Ok(event_with_room_id) => { - let _ = _tx.send(event_with_room_id); + let _ = _tx.send(event_with_room_id.cast()); } Err(e) => { error!("Failed to attach room id to message like event: {}", e); @@ -212,9 +215,9 @@ impl MatrixDriver { let _tx = tx; // Get only all state events from the state section of the sync. let handle_state = self.room.add_event_handler(move |raw: Raw| { - match with_attached_room_id(raw.cast_ref(), &_room_id) { + match add_props_to_raw(&raw, Some(_room_id.to_owned()), None) { Ok(event_with_room_id) => { - let _ = _tx.send(event_with_room_id); + let _ = _tx.send(event_with_room_id.cast()); } Err(e) => { error!("Failed to attach room id to state event: {}", e); @@ -240,7 +243,7 @@ impl MatrixDriver { let to_device_handle = self.room.client().add_event_handler( move |raw: Raw, encryption_info: Option| { - match with_attached_encryption_flag(raw, &encryption_info) { + match add_props_to_raw(&raw, None, encryption_info.as_ref()) { Ok(ev) => { let _ = tx.send(ev); } @@ -296,35 +299,40 @@ impl EventReceiver { } } -/// Attach a room id to the event. This is needed because the widget API -/// requires the room id to be present in the event. - -fn with_attached_room_id( - raw: &Raw, - room_id: &RoomId, -) -> Result> { - // This is the only modification we need to do to the events otherwise they are - // just forwarded raw to the widget. - // This is why we do the serialization dance here to allow the optimization of - // using `BTreeMap` instead of serializing the full event. - match raw.deserialize_as::>>() { - Ok(mut ev_mut) => { - ev_mut.insert("room_id".to_owned(), serde_json::value::to_raw_value(room_id)?); - Ok(Raw::new(&ev_mut)?.cast()) - } - Err(e) => Err(Error::from(e)), - } +// `room_id` and `encryption` is the only modification we need to do to the +// events otherwise they are just forwarded raw to the widget. +// This is why we do not serialization the whole event but pass it as a raw +// value through the widget driver and only serialize here to allow potimizing +// with `serde(borrow)`. +#[derive(Deserialize, Serialize)] +struct RoomIdEncryptionSerializer<'a> { + #[serde(skip_serializing_if = "Option::is_none")] + room_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + encrypted: Option, + #[serde(flatten, borrow)] + rest: &'a RawJsonValue, } -fn with_attached_encryption_flag( - raw: Raw, - encryption_info: &Option, -) -> Result> { - match raw.deserialize_as::>>() { - Ok(mut ev_mut) => { - let encrypted = encryption_info.is_some(); - ev_mut.insert("encrypted".to_owned(), serde_json::value::to_raw_value(&encrypted)?); - Ok(Raw::new(&ev_mut)?.cast()) +/// Attach additional properties to the event. +/// +/// Attach a room id to the event. This is needed because the widget API +/// requires the room id to be present in the event. +/// +/// Attach the `ecryption` flag to the event. This is needed so the widget gets +/// informed if an event is encrypted or not. Since the client is responsible +/// for decrypting the event, there otherwise is no way for the widget to know +/// if its an encrypted (signed/trusted) event or not. +fn add_props_to_raw( + raw: &Raw, + room_id: Option, + encryption_info: Option<&EncryptionInfo>, +) -> Result> { + match raw.deserialize_as::>() { + Ok(mut event) => { + event.room_id = room_id.or(event.room_id); + event.encrypted = encryption_info.map(|_| true).or(event.encrypted); + Ok(Raw::new(&event)?.cast()) } Err(e) => Err(Error::from(e)), } From 39ccdcacd5272744bbf13a1093a2c4ce2f33617c Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 7 Apr 2025 18:59:20 +0200 Subject: [PATCH 07/13] WidgetDriver: integration tests for the toDevice feature. --- crates/matrix-sdk/src/test_utils/mocks/mod.rs | 61 ++++++ crates/matrix-sdk/tests/integration/widget.rs | 196 +++++++++++++++--- .../matrix-sdk-test/src/sync_builder/mod.rs | 11 +- 3 files changed, 233 insertions(+), 35 deletions(-) diff --git a/crates/matrix-sdk/src/test_utils/mocks/mod.rs b/crates/matrix-sdk/src/test_utils/mocks/mod.rs index f7b7274cc3d..6b3b0b34e2c 100644 --- a/crates/matrix-sdk/src/test_utils/mocks/mod.rs +++ b/crates/matrix-sdk/src/test_utils/mocks/mod.rs @@ -778,6 +778,56 @@ impl MatrixMockServer { self.mock_endpoint(mock, DeleteRoomKeysVersionEndpoint).expect_default_access_token() } + /// Creates a prebuilt mock for the `/sendToDevice` endpoint. + /// + /// This mock can be used to simulate sending to-device messages in tests. + /// # Examples + /// + /// ``` + /// # #[cfg(feature = "e2e-encryption")] + /// # { + /// # tokio_test::block_on(async { + /// use std::collections::BTreeMap; + /// use matrix_sdk::{ + /// ruma::{ + /// serde::Raw, + /// api::client::to_device::send_event_to_device::v3::Request as ToDeviceRequest, + /// to_device::DeviceIdOrAllDevices, + /// user_id,owned_device_id + /// }, + /// test_utils::mocks::MatrixMockServer, + /// }; + /// use serde_json::json; + /// + /// let mock_server = MatrixMockServer::new().await; + /// let client = mock_server.client_builder().build().await; + /// + /// mock_server.mock_send_to_device().ok().mock_once().mount().await; + /// + /// let request = ToDeviceRequest::new_raw( + /// "m.custom.event".into(), + /// "txn_id".into(), + /// BTreeMap::from([ + /// (user_id!("@alice:localhost").to_owned(), BTreeMap::from([( + /// DeviceIdOrAllDevices::AllDevices, + /// Raw::new(&ruma::events::AnyToDeviceEventContent::Dummy(ruma::events::dummy::ToDeviceDummyEventContent {})).unwrap(), + /// )])), + /// ]) + /// ); + /// + /// client + /// .send(request) + /// .await + /// .expect("We should be able to send a to-device message"); + /// # anyhow::Ok(()) }); + /// # } + /// ``` + pub fn mock_send_to_device(&self) -> MockEndpoint<'_, SendToDeviceEndpoint> { + let mock = + Mock::given(method("PUT")).and(path_regex(r"^/_matrix/client/v3/sendToDevice/.*/.*")); + self.mock_endpoint(mock, SendToDeviceEndpoint).expect_default_access_token() + } + /// Create a prebuilt mock for getting the room members in a room. /// /// # Examples @@ -2315,6 +2365,17 @@ impl<'a> MockEndpoint<'a, DeleteRoomKeysVersionEndpoint> { } } +/// A prebuilt mock for the `/sendToDevice` endpoint. +/// +/// This mock can be used to simulate sending to-device messages in tests. +pub struct SendToDeviceEndpoint; +impl<'a> MockEndpoint<'a, SendToDeviceEndpoint> { + /// Returns a successful response with default data. + pub fn ok(self) -> MatrixMock<'a> { + self.respond_with(ResponseTemplate::new(200).set_body_json(json!({}))) + } +} + /// A prebuilt mock for `GET /members` request. pub struct GetRoomMembersEndpoint; diff --git a/crates/matrix-sdk/tests/integration/widget.rs b/crates/matrix-sdk/tests/integration/widget.rs index 1b7232c176b..d4c4a85b280 100644 --- a/crates/matrix-sdk/tests/integration/widget.rs +++ b/crates/matrix-sdk/tests/integration/widget.rs @@ -85,7 +85,7 @@ async fn run_test_driver( async fn recv_message(driver_handle: &WidgetDriverHandle) -> JsonObject { let fut = pin!(driver_handle.recv()); - let msg = timeout(fut, Duration::from_secs(1)).await.unwrap(); + let msg = timeout(fut, Duration::from_secs(20)).await.unwrap(); serde_json::from_str(&msg.unwrap()).unwrap() } @@ -95,15 +95,15 @@ async fn send_request( action: &str, data: impl Serialize, ) { - let sent = driver_handle - .send(json_string!({ - "api": "fromWidget", - "widgetId": WIDGET_ID, - "requestId": request_id, - "action": action, - "data": data, - })) - .await; + let json_string = json_string!({ + "api": "fromWidget", + "widgetId": WIDGET_ID, + "requestId": request_id, + "action": action, + "data": data, + }); + println!("Json string sent from the widget {}", json_string); + let sent = driver_handle.send(json_string).await; assert!(sent); } @@ -381,6 +381,8 @@ async fn test_receive_live_events() { "org.matrix.msc2762.receive.event:m.room.message#m.text", "org.matrix.msc2762.receive.state_event:m.room.name#", "org.matrix.msc2762.receive.state_event:m.room.member#@example:localhost", + "org.matrix.msc2762.receive.state_event:m.room.member#@example:localhost", + "org.matrix.msc3819.receive.to_device:my.custom.to.device" ]), ) .await; @@ -417,32 +419,63 @@ async fn test_receive_live_events() { // set room name - matches filter #3 .add_timeline_event(f.room_name("New Room Name").sender(&BOB)), ); + // to device message - doesn't match + sync_builder.add_to_device_event(json!({ + "sender": "@alice:example.com", + "type": "m.not_matching.to.device", + "content": { + "a": "test", + } + })); + // to device message - matches filter #6 + sync_builder.add_to_device_event(json!({ + "sender": "@alice:example.com", + "type": "my.custom.to.device", + "content": { + "a": "test", + } + } + )); }) .await; - let msg = recv_message(&driver_handle).await; - assert_eq!(msg["api"], "toWidget"); - assert_eq!(msg["action"], "send_event"); - assert_eq!(msg["data"]["type"], "m.room.message"); - assert_eq!(msg["data"]["room_id"], ROOM_ID.as_str()); - assert_eq!(msg["data"]["content"]["msgtype"], "m.text"); - assert_eq!(msg["data"]["content"]["body"], "simple text message"); - - let msg = recv_message(&driver_handle).await; - assert_eq!(msg["api"], "toWidget"); - assert_eq!(msg["action"], "send_event"); - assert_eq!(msg["data"]["type"], "m.room.member"); - assert_eq!(msg["data"]["room_id"], ROOM_ID.as_str()); - assert_eq!(msg["data"]["state_key"], "@example:localhost"); - assert_eq!(msg["data"]["content"]["membership"], "join"); - assert_eq!(msg["data"]["unsigned"]["prev_content"]["membership"], "join"); + // The to device and room events are racing -> we dont know the order and just + // need to store them separately. + let mut to_device: JsonObject = JsonObject::new(); + let mut events = vec![]; + for _ in 0..4 { + let msg = recv_message(&driver_handle).await; + if msg["action"] == "send_to_device" { + to_device = msg; + } else { + events.push(msg); + } + } - let msg = recv_message(&driver_handle).await; - assert_eq!(msg["api"], "toWidget"); - assert_eq!(msg["action"], "send_event"); - assert_eq!(msg["data"]["type"], "m.room.name"); - assert_eq!(msg["data"]["sender"], BOB.as_str()); - assert_eq!(msg["data"]["content"]["name"], "New Room Name"); + assert_eq!(events[0]["api"], "toWidget"); + assert_eq!(events[0]["action"], "send_event"); + assert_eq!(events[0]["data"]["type"], "m.room.message"); + assert_eq!(events[0]["data"]["room_id"], ROOM_ID.as_str()); + assert_eq!(events[0]["data"]["content"]["msgtype"], "m.text"); + assert_eq!(events[0]["data"]["content"]["body"], "simple text message"); + + assert_eq!(events[1]["api"], "toWidget"); + assert_eq!(events[1]["action"], "send_event"); + assert_eq!(events[1]["data"]["type"], "m.room.member"); + assert_eq!(events[1]["data"]["room_id"], ROOM_ID.as_str()); + assert_eq!(events[1]["data"]["state_key"], "@example:localhost"); + assert_eq!(events[1]["data"]["content"]["membership"], "join"); + assert_eq!(events[1]["data"]["unsigned"]["prev_content"]["membership"], "join"); + + assert_eq!(events[2]["api"], "toWidget"); + assert_eq!(events[2]["action"], "send_event"); + assert_eq!(events[2]["data"]["type"], "m.room.name"); + assert_eq!(events[2]["data"]["sender"], BOB.as_str()); + assert_eq!(events[2]["data"]["content"]["name"], "New Room Name"); + + assert_eq!(to_device["api"], "toWidget"); + assert_eq!(to_device["action"], "send_to_device"); + assert_eq!(to_device["data"]["type"], "my.custom.to.device"); // No more messages from the driver assert_matches!(recv_message(&driver_handle).now_or_never(), None); @@ -816,7 +849,6 @@ async fn test_send_redaction() { ]), ) .await; - mock_server.mock_room_redact().ok(event_id!("$redact_event_id")).mock_once().mount().await; send_request( @@ -843,6 +875,104 @@ async fn test_send_redaction() { assert_eq!(redact_room_id, "!a98sd12bjh:example.org"); } +async fn send_to_device_test_helper( + request_id: &str, + data: JsonValue, + expected_response: JsonValue, + calls: u64, +) -> JsonValue { + let (_, mock_server, driver_handle) = run_test_driver(false).await; + + negotiate_capabilities( + &driver_handle, + json!([ + "org.matrix.msc3819.send.to_device:my.custom.to_device_type", + "org.matrix.msc3819.send.to_device:my.other_type" + ]), + ) + .await; + + mock_server.mock_send_to_device().ok().expect(calls).mount().await; + + send_request(&driver_handle, request_id, "send_to_device", data).await; + + // Receive the response + let msg = recv_message(&driver_handle).await; + assert_eq!(msg["api"], "fromWidget"); + assert_eq!(msg["action"], "send_to_device"); + let response = msg["response"].clone(); + assert_eq!( + serde_json::to_string(&response).unwrap(), + serde_json::to_string(&expected_response).unwrap() + ); + + response +} + +#[async_test] +async fn test_send_to_device_event() { + send_to_device_test_helper( + "id_my.custom.to_device_type", + json!({ + "type":"my.custom.to_device_type", + "encrypted": false, + "messages":{ + "@username:test.org": { + "DEVICEID": { + "param1":"test", + }, + }, + } + }), + json! {{}}, + 1, + ) + .await; +} + +#[async_test] +async fn test_error_to_device_event_no_permission() { + send_to_device_test_helper( + "id_my.unallowed_type", + json!({ + "type": "my.unallowed_type", + "encrypted": false, + "messages": { + "@username:test.org": { + "DEVICEID": { + "param1":"test", + }, + }, + } + }), + // this means the server did not get the correct event type + json! {{"error": {"message": "Not allowed to send to-device message of type: my.unallowed_type"}}}, + 0 + ) + .await; +} + +#[async_test] +async fn test_send_encrypted_to_device_event() { + send_to_device_test_helper( + "my.custom.to_device_type", + json!({ + "type": "my.custom.to_device_type", + "encrypted": true, + "messages":{ + "@username:test.org": { + "DEVICEID": { + "param1":"test", + }, + }, + } + }), + json! {{"error":{"message":"Sending encrypted to_device events is not supported by the widget driver."}}}, + 0, + ) + .await; +} + async fn negotiate_capabilities(driver_handle: &WidgetDriverHandle, caps: JsonValue) { { // Receive toWidget capabilities request diff --git a/testing/matrix-sdk-test/src/sync_builder/mod.rs b/testing/matrix-sdk-test/src/sync_builder/mod.rs index 4bfc540ec36..5a4782b2b95 100644 --- a/testing/matrix-sdk-test/src/sync_builder/mod.rs +++ b/testing/matrix-sdk-test/src/sync_builder/mod.rs @@ -8,7 +8,7 @@ use ruma::{ }, IncomingResponse, }, - events::{presence::PresenceEvent, AnyGlobalAccountDataEvent}, + events::{presence::PresenceEvent, AnyGlobalAccountDataEvent, AnyToDeviceEvent}, serde::Raw, OwnedRoomId, OwnedUserId, UserId, }; @@ -58,6 +58,7 @@ pub struct SyncResponseBuilder { batch_counter: i64, /// The device lists of the user. changed_device_lists: Vec, + to_device_events: Vec>, } impl SyncResponseBuilder { @@ -162,6 +163,12 @@ impl SyncResponseBuilder { self } + /// Add a to device event. + pub fn add_to_device_event(&mut self, event: JsonValue) -> &mut Self { + self.to_device_events.push(from_json_value(event).unwrap()); + self + } + /// Builds a sync response as a JSON Value containing the events we queued /// so far. /// @@ -191,7 +198,7 @@ impl SyncResponseBuilder { "knock": self.knocked_rooms, }, "to_device": { - "events": [] + "events": self.to_device_events, }, "presence": { "events": self.presence, From 9ff12faf9fe97ee2b96d6fd5bf5cf1168a5ddd40 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 30 Apr 2025 14:09:26 +0200 Subject: [PATCH 08/13] fixup! WidgetDriver: add matrix driver toDevice support (reading and sending events via cs api) This also hooks up the widget via the machine actions. And adds toDevice events to the subscription. --- crates/matrix-sdk/src/widget/matrix.rs | 43 +++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/crates/matrix-sdk/src/widget/matrix.rs b/crates/matrix-sdk/src/widget/matrix.rs index 9bfa1e2adbd..20b1755d9b9 100644 --- a/crates/matrix-sdk/src/widget/matrix.rs +++ b/crates/matrix-sdk/src/widget/matrix.rs @@ -38,7 +38,7 @@ use ruma::{ use serde::{Deserialize, Serialize}; use serde_json::{value::RawValue as RawJsonValue, Value}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; -use tracing::error; +use tracing::{error, info}; use super::{machine::SendEventResponse, StateKeySelector}; use crate::{event_handler::EventHandlerDropGuard, room::MessagesOptions, Error, Result, Room}; @@ -305,13 +305,13 @@ impl EventReceiver { // value through the widget driver and only serialize here to allow potimizing // with `serde(borrow)`. #[derive(Deserialize, Serialize)] -struct RoomIdEncryptionSerializer<'a> { +struct RoomIdEncryptionSerializer { #[serde(skip_serializing_if = "Option::is_none")] room_id: Option, #[serde(skip_serializing_if = "Option::is_none")] encrypted: Option, - #[serde(flatten, borrow)] - rest: &'a RawJsonValue, + #[serde(flatten)] + rest: Value, } /// Attach additional properties to the event. @@ -328,12 +328,45 @@ fn add_props_to_raw( room_id: Option, encryption_info: Option<&EncryptionInfo>, ) -> Result> { - match raw.deserialize_as::>() { + match raw.deserialize_as::() { Ok(mut event) => { event.room_id = room_id.or(event.room_id); event.encrypted = encryption_info.map(|_| true).or(event.encrypted); + info!("rest is: {:?}", event.rest); Ok(Raw::new(&event)?.cast()) } Err(e) => Err(Error::from(e)), } } +#[cfg(test)] +mod tests { + use ruma::{room_id, serde::Raw}; + use serde_json::json; + + use super::add_props_to_raw; + + #[test] + fn test_app_props_to_raw() { + let raw = Raw::new(&json!({ + "encrypted": true, + "type": "m.room.message", + "content": { + "body": "Hello world" + } + })) + .unwrap(); + let room_id = room_id!("!my_id:example.org"); + let new = add_props_to_raw(&raw, Some(room_id.to_owned()), None).unwrap(); + assert_eq!( + serde_json::to_value(new).unwrap(), + json!({ + "encrypted": true, + "room_id": "!my_id:example.org", + "type": "m.room.message", + "content": { + "body": "Hello world" + } + }) + ); + } +} From b09956d6169c1aad844eb06929d1125d4daabe80 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 7 Apr 2025 18:59:20 +0200 Subject: [PATCH 09/13] WidgetDriver: change encryption send integration test to expect successful encrytion. --- crates/matrix-sdk/tests/integration/widget.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/matrix-sdk/tests/integration/widget.rs b/crates/matrix-sdk/tests/integration/widget.rs index d4c4a85b280..901b908c1a9 100644 --- a/crates/matrix-sdk/tests/integration/widget.rs +++ b/crates/matrix-sdk/tests/integration/widget.rs @@ -967,8 +967,8 @@ async fn test_send_encrypted_to_device_event() { }, } }), - json! {{"error":{"message":"Sending encrypted to_device events is not supported by the widget driver."}}}, - 0, + json! {{}}, + 1, ) .await; } From f1facc6c32e530338420933173155b85e579561c Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 30 Apr 2025 06:53:52 +0200 Subject: [PATCH 10/13] WidgetDriver: Add `io.element.call.encryption_keys` to-device capability. This needs to be part of the send/read capabilities so that to-device keys can be used. --- bindings/matrix-sdk-ffi/src/widget.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bindings/matrix-sdk-ffi/src/widget.rs b/bindings/matrix-sdk-ffi/src/widget.rs index e918df6e293..34631067b5c 100644 --- a/bindings/matrix-sdk-ffi/src/widget.rs +++ b/bindings/matrix-sdk-ffi/src/widget.rs @@ -321,7 +321,11 @@ pub fn get_element_call_required_permissions( event_type: "org.matrix.rageshake_request".to_owned(), }, // To read and send encryption keys + WidgetEventFilter::ToDeviceWithType { + event_type: "io.element.call.encryption_keys".to_owned(), + }, // TODO change this to the appropriate to-device version once ready + // remove this once all calling supports to-device encryption WidgetEventFilter::MessageLikeWithType { event_type: "io.element.call.encryption_keys".to_owned(), }, From 67b0cf08d75a99004b84f58fa6944dc6b4d53645 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 30 Apr 2025 12:47:22 +0200 Subject: [PATCH 11/13] WidgetDriver: temp, add crypto functions that are still missing from the crypto crate in a temp crate. --- crates/matrix-sdk/src/widget/matrix.rs | 144 ++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 3 deletions(-) diff --git a/crates/matrix-sdk/src/widget/matrix.rs b/crates/matrix-sdk/src/widget/matrix.rs index 20b1755d9b9..9a97d8015db 100644 --- a/crates/matrix-sdk/src/widget/matrix.rs +++ b/crates/matrix-sdk/src/widget/matrix.rs @@ -17,6 +17,7 @@ use std::collections::BTreeMap; +use futures_util::future::join_all; use matrix_sdk_base::deserialized_responses::{EncryptionInfo, RawAnySyncOrStrippedState}; use ruma::{ api::client::{ @@ -273,9 +274,36 @@ impl MatrixDriver { let client = self.room.client(); let request = if encrypted { - return Err(Error::UnknownError( - "Sending encrypted to_device events is not supported by the widget driver.".into(), - )); + // We first want to get all missing session before we start any to device + // sending! + client.claim_one_time_keys(messages.keys().map(|u| u.as_ref())).await?; + let encrypted_content: BTreeMap< + OwnedUserId, + BTreeMap>, + > = join_all(messages.into_iter().map(|(user_id, device_content_map)| { + let event_type = event_type.clone(); + async move { + ( + user_id.clone(), + to_device_crypto::encrypted_device_content_map( + &self.room.client(), + &user_id, + &event_type, + device_content_map, + ) + .await, + ) + } + })) + .await + .into_iter() + .collect(); + + RumaToDeviceRequest::new_raw( + ToDeviceEventType::RoomEncrypted, + TransactionId::new(), + encrypted_content, + ) } else { RumaToDeviceRequest::new_raw(event_type, TransactionId::new(), messages) }; @@ -338,6 +366,116 @@ fn add_props_to_raw( Err(e) => Err(Error::from(e)), } } + +/// Move this into the `matrix_crypto` crate! +/// This module contains helper functions to encrypt to device events. +mod to_device_crypto { + use std::collections::BTreeMap; + + use futures_util::future::join_all; + use ruma::{ + events::{AnyToDeviceEventContent, ToDeviceEventType}, + serde::Raw, + to_device::DeviceIdOrAllDevices, + UserId, + }; + use serde_json::Value; + use tracing::{info, warn}; + + use crate::{encryption::identities::Device, executor::spawn, Client, Error, Result}; + + /// This encrypts to device content for a collection of devices. + /// It will ignore all devices where errors occurred or where the device + /// is not verified or where th user has a has_verification_violation. + async fn encrypted_content_for_devices( + unencrypted_content: &Raw, + devices: Vec, + event_type: &ToDeviceEventType, + ) -> Result)>> { + let content: Value = unencrypted_content.deserialize_as().map_err(Into::::into)?; + let event_type = event_type.clone(); + let device_content_tasks = devices.into_iter().map(|device| spawn({ + let event_type = event_type.clone(); + let content = content.clone(); + + async move { + if !device.is_cross_signed_by_owner() { + info!("Device {} is not verified, skipping encryption", device.device_id()); + return None; + } + match device + .inner + .encrypt_event_raw(&event_type.to_string(), &content) + .await { + Ok(encrypted) => Some((device.device_id().to_owned().into(), encrypted.cast())), + Err(e) =>{ info!("Failed to encrypt to_device event from widget for device: {} because, {}", device.device_id(), e); None}, + } + } + })); + let device_encrypted_content_map = + join_all(device_content_tasks).await.into_iter().flatten().flatten(); + Ok(device_encrypted_content_map) + } + + /// Convert the device content map for one user into the same content + /// map with encrypted content This needs to flatten the vectors + /// we get from `encrypted_content_for_devices` + /// since one `DeviceIdOrAllDevices` id can be multiple devices. + pub(super) async fn encrypted_device_content_map( + client: &Client, + user_id: &UserId, + event_type: &ToDeviceEventType, + device_content_map: BTreeMap>, + ) -> BTreeMap> { + let device_map_futures = + device_content_map.into_iter().map(|(device_or_all_id, content)| spawn({ + let client = client.clone(); + let user_id = user_id.to_owned(); + let event_type = event_type.clone(); + async move { + let Ok(user_devices) = client.encryption().get_user_devices(&user_id).await else { + warn!("Failed to get user devices for user: {}", user_id); + return None; + }; + let Ok(user_identity) = client.encryption().get_user_identity(&user_id).await else{ + warn!("Failed to get user identity for user: {}", user_id); + return None; + }; + if user_identity.map(|i|i.has_verification_violation()).unwrap_or(false) { + info!("User {} has a verification violation, skipping encryption", user_id); + return None; + } + let devices: Vec = match device_or_all_id { + DeviceIdOrAllDevices::DeviceId(device_id) => { + vec![user_devices.get(&device_id)].into_iter().flatten().collect() + } + DeviceIdOrAllDevices::AllDevices => user_devices.devices().collect(), + }; + encrypted_content_for_devices( + &content, + devices, + &event_type, + ) + .await + .map_err(|e| info!("WidgetDriver: could not encrypt content for to device widget event content: {}. because, {}", content.json(), e)) + .ok() + }})); + let content_map_iterator = join_all(device_map_futures).await.into_iter(); + + // The first flatten takes the iterator over Result)>>, JoinError>> + // and flattens the Result (drops Err() items) + // The second takes the iterator over: Option)>> + // and flattens the Option (drops None items) + // The third takes the iterator over iterators: impl Iterator)> + // and flattens it to just an iterator over (DeviceIdOrAllDevices, + // Raw) + content_map_iterator.flatten().flatten().flatten().collect() + } +} + #[cfg(test)] mod tests { use ruma::{room_id, serde::Raw}; From 8d0b69eb83d61933d627f382c6642602ae7fad56 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 2 May 2025 11:11:11 +0200 Subject: [PATCH 12/13] rename to-device ffi filter to match declaration --- bindings/matrix-sdk-ffi/src/widget.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bindings/matrix-sdk-ffi/src/widget.rs b/bindings/matrix-sdk-ffi/src/widget.rs index 34631067b5c..214f94e3d8a 100644 --- a/bindings/matrix-sdk-ffi/src/widget.rs +++ b/bindings/matrix-sdk-ffi/src/widget.rs @@ -321,9 +321,7 @@ pub fn get_element_call_required_permissions( event_type: "org.matrix.rageshake_request".to_owned(), }, // To read and send encryption keys - WidgetEventFilter::ToDeviceWithType { - event_type: "io.element.call.encryption_keys".to_owned(), - }, + WidgetEventFilter::ToDevice { event_type: "io.element.call.encryption_keys".to_owned() }, // TODO change this to the appropriate to-device version once ready // remove this once all calling supports to-device encryption WidgetEventFilter::MessageLikeWithType { From 35501dfe8a8a61dcc5c7490bce24ad4101a7f25e Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 2 May 2025 11:11:37 +0200 Subject: [PATCH 13/13] Skip device verification checks for sending (this would otherwise exclude spa guests) --- crates/matrix-sdk/src/widget/matrix.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/crates/matrix-sdk/src/widget/matrix.rs b/crates/matrix-sdk/src/widget/matrix.rs index 9a97d8015db..31a067d3162 100644 --- a/crates/matrix-sdk/src/widget/matrix.rs +++ b/crates/matrix-sdk/src/widget/matrix.rs @@ -399,10 +399,11 @@ mod to_device_crypto { let content = content.clone(); async move { - if !device.is_cross_signed_by_owner() { - info!("Device {} is not verified, skipping encryption", device.device_id()); - return None; - } + // This is not yet used. It is incompatible with the spa guest mode (the spa will not verify its crypto identity) + // if !device.is_cross_signed_by_owner() { + // info!("Device {} is not verified, skipping encryption", device.device_id()); + // return None; + // } match device .inner .encrypt_event_raw(&event_type.to_string(), &content) @@ -437,14 +438,15 @@ mod to_device_crypto { warn!("Failed to get user devices for user: {}", user_id); return None; }; - let Ok(user_identity) = client.encryption().get_user_identity(&user_id).await else{ - warn!("Failed to get user identity for user: {}", user_id); - return None; - }; - if user_identity.map(|i|i.has_verification_violation()).unwrap_or(false) { - info!("User {} has a verification violation, skipping encryption", user_id); - return None; - } + // This is not yet used. It is incompatible with the spa guest mode (the spa will not verify its crypto identity) + // let Ok(user_identity) = client.encryption().get_user_identity(&user_id).await else{ + // warn!("Failed to get user identity for user: {}", user_id); + // return None; + // }; + // if user_identity.map(|i|i.has_verification_violation()).unwrap_or(false) { + // info!("User {} has a verification violation, skipping encryption", user_id); + // return None; + // } let devices: Vec = match device_or_all_id { DeviceIdOrAllDevices::DeviceId(device_id) => { vec![user_devices.get(&device_id)].into_iter().flatten().collect()