commit b1cc1f9d863aa9e7afdc7aebd8a7377a361487e8
author hui su <huisu@google.com> Tue Apr 11 17:41:29 2017 -0700
committer Hui Su <huisu@google.com> Wed Apr 12 04:37:00 2017 +0000
tree 3e2d6db500c2ba58d980c428ad55090a887faafd
parent 18ad894a23e7ee53019c1d90c48a8a906dc1d04e

Fix a bug for high-bit-depth in av1_dist_block()

BUG=aomedia:426

Change-Id: I246b0de3bd25863853643bdb238899bf8e5847be

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index f61cb55..6c8154d 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1459,6 +1459,10 @@
       const int16_t *diff = &p->src_diff[diff_idx];
       tmp = sum_squares_visible(xd, plane, diff, diff_stride, blk_row, blk_col,
                                 plane_bsize, tx_bsize);
+#if CONFIG_AOM_HIGHBITDEPTH
+      if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
+        tmp = ROUND_POWER_OF_TWO(tmp, (xd->bd - 8) * 2);
+#endif  // CONFIG_AOM_HIGHBITDEPTH
     }
     *out_sse = (int64_t)tmp * 16;