commit 8b65d76f2f95b1bfdc82b8f135246cb367d444e6
author Wan-Teh Chang <wtc@google.com> Tue Sep 12 16:38:47 2023 -0700
committer Wan-Teh Chang <wtc@google.com> Wed Sep 13 19:30:24 2023 +0000
tree 2d2f82e0e31805c0dd9b8ff729a5713310d534dd
parent 89795ce03a7196ac8be9b8f207d7a536b7947680

Fix byte order issue in write_enc_data_to_out_buf

Fix the byte order issue for big-endian systems introduced in
https://aomedia-review.googlesource.com/c/aom/+/167581.

Use the existing HToBE64 macro defined in aom_util/endian_inl.h and
delete the new get_byteswap64 function from aom_ports/bitops.h.

Based on the patch in bug aomedia:3487.

Bug: aomedia:3487
Change-Id: I56183e062a9f2b18c4c6158fd8e47c5062de85ee

aom_dsp/entenc.h

diff --git a/aom_dsp/entenc.h b/aom_dsp/entenc.h
index 467e47b..d26f027 100644
--- a/aom_dsp/entenc.h
+++ b/aom_dsp/entenc.h
@@ -13,7 +13,7 @@
 #define AOM_AOM_DSP_ENTENC_H_
 #include <stddef.h>
 #include "aom_dsp/entcode.h"
-#include "aom_ports/bitops.h"
+#include "aom_util/endian_inl.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -87,13 +87,14 @@
   } while (carry);
 }
 
-// Reverse byte order and write data to buffer adding the carry-bit
+// Convert to big-endian byte order and write data to buffer adding the
+// carry-bit
 static AOM_INLINE void write_enc_data_to_out_buf(unsigned char *out,
                                                  uint32_t offs, uint64_t output,
                                                  uint64_t carry,
                                                  uint32_t *enc_offs,
                                                  uint8_t num_bytes_ready) {
-  const uint64_t reg = get_byteswap64(output) >> ((8 - num_bytes_ready) << 3);
+  const uint64_t reg = HToBE64(output << ((8 - num_bytes_ready) << 3));
   memcpy(&out[offs], &reg, 8);
   // Propagate carry backwards if exists
   if (carry) {

aom_ports/bitops.h

diff --git a/aom_ports/bitops.h b/aom_ports/bitops.h
index 3c5b992..7f4c165 100644
--- a/aom_ports/bitops.h
+++ b/aom_ports/bitops.h
@@ -13,7 +13,6 @@
 #define AOM_AOM_PORTS_BITOPS_H_
 
 #include <assert.h>
-#include <stdint.h>
 
 #include "aom_ports/msvc.h"
 #include "config/aom_config.h"
@@ -34,12 +33,8 @@
 // These versions of get_msb() are only valid when n != 0 because all
 // of the optimized versions are undefined when n == 0:
 
-// get_byteswap64:
-// Returns the number (uint64_t) with byte-positions reversed
-// e.g. input 0x123456789ABCDEF0 returns 0xF0DEBC9A78563412
-
 // GCC compiler: https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
-// MSVC: https://learn.microsoft.com/en-us/cpp/c-runtime-library/
+// MSVC: https://learn.microsoft.com/en-us/cpp/intrinsics/compiler-intrinsics
 
 // use GNU builtins where available.
 #if defined(__GNUC__) && \
@@ -48,10 +43,6 @@
   assert(n != 0);
   return 31 ^ __builtin_clz(n);
 }
-
-static INLINE uint64_t get_byteswap64(uint64_t num) {
-  return __builtin_bswap64(num);
-}
 #elif defined(USE_MSC_INTRINSICS)
 #pragma intrinsic(_BitScanReverse)
 
@@ -61,10 +52,6 @@
   _BitScanReverse(&first_set_bit, n);
   return first_set_bit;
 }
-
-static INLINE uint64_t get_byteswap64(uint64_t num) {
-  return _byteswap_uint64(num);
-}
 #undef USE_MSC_INTRINSICS
 #else
 static INLINE int get_msb(unsigned int n) {
@@ -82,26 +69,6 @@
   }
   return log;
 }
-
-static INLINE uint64_t get_byteswap64(uint64_t num) {
-  uint64_t out = 0x00;
-  uint64_t mask = 0xFF00000000000000;
-  int bit_shift = 56;  // 7 bytes
-  // 4 ms bytes
-  do {
-    out |= (num & mask) >> bit_shift;
-    mask >>= 8;
-    bit_shift -= 16;
-  } while (bit_shift >= 0);
-  // 4 ls bytes
-  bit_shift = 8;  // 1 byte
-  do {
-    out |= (num & mask) << bit_shift;
-    mask >>= 8;
-    bit_shift += 16;
-  } while (bit_shift <= 56);
-  return out;
-}
 #endif
 
 #ifdef __cplusplus