commit d18c84cb613286ca5c76fe5b745c3bbf7a9f863e
author Jingning Han <jingning@google.com> Thu Apr 27 11:25:29 2017 -0700
committer Jingning Han <jingning@google.com> Thu Apr 27 21:10:58 2017 +0000
tree d086b643de6a37286492410b9e40fd120928ae13
parent 0a6731f52c9f42aa9fcfe901e6efc80c008aed11

Fix quantize_b_c function

Correctly compute the quantized coefficient index. This resolves
an enc/dec mismatch issue when the c function is in use.

Change-Id: I670b5a18310fa68c994f43b6a8bc9b0d8ff907ae

aom_dsp/quantize.c

diff --git a/aom_dsp/quantize.c b/aom_dsp/quantize.c
index c82841a..0759c22 100644
--- a/aom_dsp/quantize.c
+++ b/aom_dsp/quantize.c
@@ -83,7 +83,7 @@
                        quant_shift_ptr[rc != 0]) >>
                       (16 - log_scale));  // quantization
 #endif  // CONFIG_AOM_QM
-        qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
+        qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
 #if CONFIG_AOM_QM
         const int dequant =
             (dequant_ptr[rc != 0] * iqm_ptr[rc] + (1 << (AOM_QM_BITS - 1))) >>