commit b433f4c40bbcdbaa88258445116a9850b49d5dc1
author Jingning Han <jingning@google.com> Fri Nov 17 15:43:59 2017 -0800
committer Jingning Han <jingning@google.com> Sat Nov 18 21:15:53 2017 +0000
tree c904cd77bae0ca8f6fbe8c4e776316cae35aadf2
parent ea636197d6cb43834eb70044f1733b35c0d73bea

Properly scale the distortion in lv-map optimize_b

Account for the bit-depth in calculating the distortion for lv-map
coefficient level optimization.

Change-Id: I47c60f15bfda5aeba8e32f1ac898015650a057db

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 30a56e5..2e1e1ed 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -2061,7 +2061,9 @@
 
   const int shift = av1_get_tx_scale(tx_size);
   const int64_t rdmult =
-      (x->rdmult * plane_rd_mult[is_inter][plane_type] + 2) >> 2;
+      ((x->rdmult * plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8))) +
+       2) >>
+      2;
   uint8_t levels_buf[TX_PAD_2D];
   uint8_t *const levels = set_levels(levels_buf, width);