Skip to content

fix(server): use desktop size for RFX channel size #756

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 17, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions crates/ironrdp-server/src/encoder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ impl UpdateEncoder {
BitmapUpdater::Bitmap(BitmapHandler::new())
} else if remotefx.is_some() {
let (algo, id) = remotefx.unwrap();
BitmapUpdater::RemoteFx(RemoteFxHandler::new(algo, id))
BitmapUpdater::RemoteFx(RemoteFxHandler::new(algo, id, desktop_size))
} else {
BitmapUpdater::None(NoneHandler)
};
Expand All @@ -68,6 +68,7 @@ impl UpdateEncoder {

pub(crate) fn set_desktop_size(&mut self, size: DesktopSize) {
self.desktop_size = size;
self.bitmap_updater.set_desktop_size(size);
}

fn rgba_pointer(ptr: RGBAPointer) -> Result<UpdateFragmenter> {
Expand Down Expand Up @@ -181,6 +182,12 @@ impl BitmapUpdater {
Self::RemoteFx(up) => up.handle(bitmap),
}
}

fn set_desktop_size(&mut self, size: DesktopSize) {
if let Self::RemoteFx(up) = self {
up.set_desktop_size(size)
}
}
}

trait BitmapUpdateHandler {
Expand Down Expand Up @@ -246,28 +253,36 @@ impl BitmapUpdateHandler for BitmapHandler {
struct RemoteFxHandler {
remotefx: RfxEncoder,
codec_id: u8,
desktop_size: Option<DesktopSize>,
}

impl RemoteFxHandler {
fn new(algo: EntropyBits, codec_id: u8) -> Self {
fn new(algo: EntropyBits, codec_id: u8, desktop_size: DesktopSize) -> Self {
Self {
remotefx: RfxEncoder::new(algo),
desktop_size: Some(desktop_size),
codec_id,
}
}

fn set_desktop_size(&mut self, size: DesktopSize) {
self.desktop_size = Some(size);
}
}

impl BitmapUpdateHandler for RemoteFxHandler {
fn handle(&mut self, bitmap: &BitmapUpdate) -> Result<UpdateFragmenter> {
let mut buffer = vec![0; bitmap.data.len()];
let len = loop {
match self.remotefx.encode(bitmap, buffer.as_mut_slice()) {
match self
.remotefx
.encode(bitmap, buffer.as_mut_slice(), self.desktop_size.take())
{
Err(e) => match e.kind() {
ironrdp_core::EncodeErrorKind::NotEnoughBytes { .. } => {
buffer.resize(buffer.len() * 2, 0);
debug!("encoder buffer resized to: {}", buffer.len() * 2);
}

_ => Err(e).context("RemoteFX encode error")?,
},
Ok(len) => break len,
Expand Down
44 changes: 25 additions & 19 deletions crates/ironrdp-server/src/encoder/rfx.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use ironrdp_acceptor::DesktopSize;
use ironrdp_core::{cast_length, other_err, Encode, EncodeResult};
use ironrdp_graphics::color_conversion::to_64x64_ycbcr_tile;
use ironrdp_graphics::rfx_encode_component;
Expand Down Expand Up @@ -25,29 +26,34 @@ impl RfxEncoder {
Self { entropy_algorithm }
}

pub(crate) fn encode(&mut self, bitmap: &BitmapUpdate, output: &mut [u8]) -> EncodeResult<usize> {
pub(crate) fn encode(
&mut self,
bitmap: &BitmapUpdate,
output: &mut [u8],
desktop_size: Option<DesktopSize>,
) -> EncodeResult<usize> {
let mut cursor = WriteCursor::new(output);

let width = bitmap.width.get();
let height = bitmap.height.get();
let entropy_algorithm = self.entropy_algorithm;

// header messages
// FIXME: skip if unnecessary?
Block::Sync(SyncPdu).encode(&mut cursor)?;
let context = rfx::ContextPdu {
flags: OperatingMode::IMAGE_MODE,
entropy_algorithm,
};
Block::CodecChannel(CodecChannel::Context(context)).encode(&mut cursor)?;

let channels = ChannelsPdu(vec![RfxChannel {
width: cast_length!("width", width)?,
height: cast_length!("height", height)?,
}]);
Block::Channels(channels).encode(&mut cursor)?;

Block::CodecVersions(CodecVersionsPdu).encode(&mut cursor)?;
if let Some(desktop_size) = desktop_size {
let width = desktop_size.width;
let height = desktop_size.height;
Block::Sync(SyncPdu).encode(&mut cursor)?;
let context = rfx::ContextPdu {
flags: OperatingMode::IMAGE_MODE,
entropy_algorithm,
};
Block::CodecChannel(CodecChannel::Context(context)).encode(&mut cursor)?;

let channels = ChannelsPdu(vec![RfxChannel {
width: cast_length!("width", width)?,
height: cast_length!("height", height)?,
}]);
Block::Channels(channels).encode(&mut cursor)?;

Block::CodecVersions(CodecVersionsPdu).encode(&mut cursor)?;
}

// data messages
let frame_begin = FrameBeginPdu {
Expand Down
Loading