commit 5cb408e55b1efeb79034534f27e1cab9a5943703
author Jingning Han <jingning@google.com> Fri Nov 17 14:43:39 2017 -0800
committer Jingning Han <jingning@google.com> Sat Nov 18 21:15:44 2017 +0000
tree d7340b896364dd4cd0511d4d2530c70d154d09f6
parent f40a957731c8d2cad2e425cc768a5bb18709e6e7

Fix av1_txb_init_levels clamping limit

This solves an enc/dec mismatch issue due to the fact that
SIMD implementation compares signed int8_t. Hence we need to
limit the base level reference to be under 127, to avoid cmpt
instruction taking it as a negative number.

BUG=aomedia:1045

Change-Id: I2ed616fda52415fbf50451660b9564df1adf77af

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index fc1e236..30a56e5 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -277,7 +277,7 @@
                                  const TxbInfo *const txb_info) {
   txb_info->qcoeff[coeff_idx] = qc;
   txb_info->levels[get_paded_idx(coeff_idx, txb_info->bwl)] =
-      (uint8_t)clamp(abs(qc), 0, UINT8_MAX);
+      (uint8_t)clamp(abs(qc), 0, INT8_MAX);
 }
 
 static INLINE void update_coeff(const int coeff_idx, const tran_low_t qc,
@@ -300,7 +300,7 @@
 
   for (int i = 0; i < height; i++) {
     for (int j = 0; j < width; j++) {
-      *ls++ = (uint8_t)clamp(abs(coeff[i * width + j]), 0, UINT8_MAX);
+      *ls++ = (uint8_t)clamp(abs(coeff[i * width + j]), 0, INT8_MAX);
     }
     for (int j = 0; j < TX_PAD_HOR; j++) {
       *ls++ = 0;