commit d43d6777a69348b8dc3a6ac58c6a8639d24b1b01
author Yaowu Xu <yaowu@google.com> Mon Jun 26 15:48:37 2017 -0700
committer Yaowu Xu <yaowu@google.com> Mon Jun 26 15:49:37 2017 -0700
tree 424788f7e7a42ce32a3f8d03228133fb3086dbf2
parent 284c883029eda1c96636f79c1745d476789a0f32

quantize.c: convert to int before apply sign

This change makes the conversions similar to those in av1_quantize.c,
and fix ubsan warnings shown in nightly tests.

Change-Id: I90851a80dcb9f052a32bf22199fd9ef8ff927725

aom_dsp/quantize.c

diff --git a/aom_dsp/quantize.c b/aom_dsp/quantize.c
index 0759c22..d5c89eb 100644
--- a/aom_dsp/quantize.c
+++ b/aom_dsp/quantize.c
@@ -716,8 +716,7 @@
       if (abs_coeff >= zbins[rc != 0]) {
         const int64_t tmp1 = abs_coeff + round_ptr[rc != 0];
         const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
-        const uint32_t abs_qcoeff =
-            (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);
+        const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);
         qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
         if (abs_qcoeff) eob = i;
@@ -767,8 +766,7 @@
       const int64_t tmp1 =
           abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);
       const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
-      const uint32_t abs_qcoeff =
-          (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);
+      const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);
       qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
       if (abs_qcoeff) eob = idx_arr[i];
@@ -818,8 +816,7 @@
       const int64_t tmp1 =
           abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 2);
       const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
-      const uint32_t abs_qcoeff =
-          (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 14);
+      const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 14);
       qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 4;
       if (abs_qcoeff) eob = idx_arr[i];