https://github.com/FreeRDP/FreeRDP/issues/9616
From 8b349f4f2b5d14e28a6a73e9c2b9860d7f275bc4 Mon Sep 17 00:00:00 2001
From: Mike Gilbert <floppym@gentoo.org>
Date: Thu, 7 Dec 2023 15:54:38 -0500
Subject: [PATCH] Revert "[codec] encode messages considering endianness"
This reverts commit e3ef8a02ddf079bcf6dfa58888db69a709ce50c9.
---
libfreerdp/codec/nsc_encode.c | 76 +++++++++++++----------------------
libfreerdp/codec/rfx_encode.c | 74 +++++++++++++---------------------
2 files changed, 58 insertions(+), 92 deletions(-)
diff --git a/libfreerdp/codec/nsc_encode.c b/libfreerdp/codec/nsc_encode.c
index 477318cc1..fd9626a6d 100644
--- a/libfreerdp/codec/nsc_encode.c
+++ b/libfreerdp/codec/nsc_encode.c
@@ -119,8 +119,6 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, UI
UINT16 rw;
BYTE ccl;
const BYTE* src;
- const UINT32* src_32;
- const UINT16* src_16;
BYTE* yplane = NULL;
BYTE* coplane = NULL;
BYTE* cgplane = NULL;
@@ -142,85 +140,69 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, UI
coplane = context->priv->PlaneBuffers[1] + y * rw;
cgplane = context->priv->PlaneBuffers[2] + y * rw;
aplane = context->priv->PlaneBuffers[3] + y * context->width;
- src_32 = (UINT32*)src;
- src_16 = (UINT16*)src;
for (x = 0; x < context->width; x++)
{
switch (context->format)
{
case PIXEL_FORMAT_BGRX32:
- b_val = (INT16)(*src_32 & 0xFF);
- g_val = (INT16)((*src_32 >> 8) & 0xFF);
- r_val = (INT16)((*src_32 >> 16) & 0xFF);
+ b_val = *src++;
+ g_val = *src++;
+ r_val = *src++;
+ src++;
a_val = 0xFF;
- src_32++;
break;
case PIXEL_FORMAT_BGRA32:
- b_val = (INT16)(*src_32 & 0xFF);
- g_val = (INT16)((*src_32 >> 8) & 0xFF);
- r_val = (INT16)((*src_32 >> 16) & 0xFF);
- a_val = (INT16)((*src_32 >> 24) & 0xFF);
- src_32++;
+ b_val = *src++;
+ g_val = *src++;
+ r_val = *src++;
+ a_val = *src++;
break;
case PIXEL_FORMAT_RGBX32:
- r_val = (INT16)(*src_32 & 0xFF);
- g_val = (INT16)((*src_32 >> 8) & 0xFF);
- b_val = (INT16)((*src_32 >> 16) & 0xFF);
+ r_val = *src++;
+ g_val = *src++;
+ b_val = *src++;
+ src++;
a_val = 0xFF;
- src_32++;
break;
case PIXEL_FORMAT_RGBA32:
- r_val = (INT16)(*src_32 & 0xFF);
- g_val = (INT16)((*src_32 >> 8) & 0xFF);
- b_val = (INT16)((*src_32 >> 16) & 0xFF);
- a_val = (INT16)((*src_32 >> 24) & 0xFF);
- src_32++;
+ r_val = *src++;
+ g_val = *src++;
+ b_val = *src++;
+ a_val = *src++;
break;
case PIXEL_FORMAT_BGR24:
-#ifdef __LITTLE_ENDIAN__
b_val = *src++;
g_val = *src++;
r_val = *src++;
-#else
- r_val = *src++;
- g_val = *src++;
- b_val = *src++;
-#endif
a_val = 0xFF;
break;
case PIXEL_FORMAT_RGB24:
-#ifdef __LITTLE_ENDIAN__
r_val = *src++;
g_val = *src++;
b_val = *src++;
-#else
- b_val = *src++;
- g_val = *src++;
- r_val = *src++;
-#endif
a_val = 0xFF;
break;
case PIXEL_FORMAT_BGR16:
- b_val = (INT16)((*src_16) & 0x1F);
- g_val = (INT16)((*src_16 >> 5) & 0x3F);
- r_val = (INT16)((*src_16 >> 11) & 0x1F);
+ b_val = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5));
+ g_val = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3));
+ r_val = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07));
a_val = 0xFF;
- src_16++;
+ src += 2;
break;
case PIXEL_FORMAT_RGB16:
- r_val = (INT16)((*src_16) & 0x1F);
- g_val = (INT16)((*src_16 >> 5) & 0x3F);
- b_val = (INT16)((*src_16 >> 11) & 0x1F);
+ r_val = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5));
+ g_val = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3));
+ b_val = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07));
a_val = 0xFF;
- src_16++;
+ src += 2;
break;
case PIXEL_FORMAT_A4:
@@ -228,17 +210,17 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, UI
int shift;
BYTE idx;
shift = (7 - (x % 8));
- idx = (BYTE)(((*src_32 & 0xFF) >> shift) & 1);
- idx |= (BYTE)(((((*src_32 >> 8) & 0xFF) >> shift) & 1) << 1);
- idx |= (BYTE)(((((*src_32 >> 16) & 0xFF) >> shift) & 1) << 2);
- idx |= (BYTE)(((((*src_32 >> 24) & 0xFF) >> shift) & 1) << 3);
+ idx = ((*src) >> shift) & 1;
+ idx |= (((*(src + 1)) >> shift) & 1) << 1;
+ idx |= (((*(src + 2)) >> shift) & 1) << 2;
+ idx |= (((*(src + 3)) >> shift) & 1) << 3;
idx *= 3;
r_val = (INT16)context->palette[idx];
g_val = (INT16)context->palette[idx + 1];
b_val = (INT16)context->palette[idx + 2];
if (shift == 0)
- src_32++;
+ src += 4;
}
a_val = 0xFF;
diff --git a/libfreerdp/codec/rfx_encode.c b/libfreerdp/codec/rfx_encode.c
index 355f74340..b4f393605 100644
--- a/libfreerdp/codec/rfx_encode.c
+++ b/libfreerdp/codec/rfx_encode.c
@@ -49,8 +49,6 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
int x_exceed;
int y_exceed;
const BYTE* src;
- const UINT32* src_32;
- const UINT16* src_16;
INT16 r, g, b;
INT16 *r_last, *g_last, *b_last;
x_exceed = 64 - width;
@@ -59,8 +57,6 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
for (y = 0; y < height; y++)
{
src = rgb_data + y * rowstride;
- src_32 = (UINT32*)src;
- src_16 = (UINT16*)src;
switch (pixel_format)
{
@@ -68,10 +64,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_BGRA32:
for (x = 0; x < width; x++)
{
- *b_buf++ = (INT16)(*src_32 & 0xFF);
- *g_buf++ = (INT16)((*src_32 >> 8) & 0xFF);
- *r_buf++ = (INT16)((*src_32 >> 16) & 0xFF);
- src_32++;
+ *b_buf++ = (INT16)(*src++);
+ *g_buf++ = (INT16)(*src++);
+ *r_buf++ = (INT16)(*src++);
+ src++;
}
break;
@@ -80,10 +76,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_ABGR32:
for (x = 0; x < width; x++)
{
- *b_buf++ = (INT16)((*src_32 >> 8) & 0xFF);
- *g_buf++ = (INT16)((*src_32 >> 16) & 0xFF);
- *r_buf++ = (INT16)((*src_32 >> 24) & 0xFF);
- src_32++;
+ src++;
+ *b_buf++ = (INT16)(*src++);
+ *g_buf++ = (INT16)(*src++);
+ *r_buf++ = (INT16)(*src++);
}
break;
@@ -92,10 +88,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_RGBA32:
for (x = 0; x < width; x++)
{
- *r_buf++ = (INT16)(*src_32 & 0xFF);
- *g_buf++ = (INT16)((*src_32 >> 8) & 0xFF);
- *b_buf++ = (INT16)((*src_32 >> 16) & 0xFF);
- src_32++;
+ *r_buf++ = (INT16)(*src++);
+ *g_buf++ = (INT16)(*src++);
+ *b_buf++ = (INT16)(*src++);
+ src++;
}
break;
@@ -104,10 +100,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_ARGB32:
for (x = 0; x < width; x++)
{
- *r_buf++ = (INT16)((*src_32 >> 8) & 0xFF);
- *g_buf++ = (INT16)((*src_32 >> 16) & 0xFF);
- *b_buf++ = (INT16)((*src_32 >> 24) & 0xFF);
- src_32++;
+ src++;
+ *r_buf++ = (INT16)(*src++);
+ *g_buf++ = (INT16)(*src++);
+ *b_buf++ = (INT16)(*src++);
}
break;
@@ -115,15 +111,9 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_BGR24:
for (x = 0; x < width; x++)
{
-#ifdef __LITTLE_ENDIAN__
*b_buf++ = (INT16)(*src++);
*g_buf++ = (INT16)(*src++);
*r_buf++ = (INT16)(*src++);
-#else
- *r_buf++ = (INT16)(*src++);
- *g_buf++ = (INT16)(*src++);
- *b_buf++ = (INT16)(*src++);
-#endif
}
break;
@@ -131,15 +121,9 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_RGB24:
for (x = 0; x < width; x++)
{
-#ifdef __LITTLE_ENDIAN__
*r_buf++ = (INT16)(*src++);
*g_buf++ = (INT16)(*src++);
*b_buf++ = (INT16)(*src++);
-#else
- *b_buf++ = (INT16)(*src++);
- *g_buf++ = (INT16)(*src++);
- *r_buf++ = (INT16)(*src++);
-#endif
}
break;
@@ -147,10 +131,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_BGR16:
for (x = 0; x < width; x++)
{
- *b_buf++ = (INT16)((*src_16) & 0x1F);
- *g_buf++ = (INT16)((*src_16 >> 5) & 0x3F);
- *r_buf++ = (INT16)((*src_16 >> 11) & 0x1F);
- src_16++;
+ *b_buf++ = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5));
+ *g_buf++ = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3));
+ *r_buf++ = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07));
+ src += 2;
}
break;
@@ -158,10 +142,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_RGB16:
for (x = 0; x < width; x++)
{
- *r_buf++ = (INT16)((*src_16 & 0x1F));
- *g_buf++ = (INT16)((*src_16 >> 5) & 0x3F);
- *b_buf++ = (INT16)((*src_16 >> 11) & 0x1F);
- src_16++;
+ *r_buf++ = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5));
+ *g_buf++ = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3));
+ *b_buf++ = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07));
+ src += 2;
}
break;
@@ -175,17 +159,17 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
int shift;
BYTE idx;
shift = (7 - (x % 8));
- idx = (BYTE)(((*src_32 & 0xFF) >> shift) & 1);
- idx |= (BYTE)(((((*src_32 >> 8) & 0xFF) >> shift) & 1) << 1);
- idx |= (BYTE)(((((*src_32 >> 16) & 0xFF) >> shift) & 1) << 2);
- idx |= (BYTE)(((((*src_32 >> 24) & 0xFF) >> shift) & 1) << 3);
+ idx = ((*src) >> shift) & 1;
+ idx |= (((*(src + 1)) >> shift) & 1) << 1;
+ idx |= (((*(src + 2)) >> shift) & 1) << 2;
+ idx |= (((*(src + 3)) >> shift) & 1) << 3;
idx *= 3;
*r_buf++ = (INT16)palette[idx];
*g_buf++ = (INT16)palette[idx + 1];
*b_buf++ = (INT16)palette[idx + 2];
if (shift == 0)
- src_32++;
+ src += 4;
}
break;
--
2.43.0