commit f986af1ce77166210a5373ef039407e5aee31f10
author Yushin Cho <ycho@mozilla.com> Mon Nov 20 15:22:43 2017 -0800
committer Yushin Cho <ycho@mozilla.com> Wed Nov 29 01:22:23 2017 +0000
tree 2930add43529dc4ff6da4b2eb4a5f8151dfc2e61
parent 19619886c48b453c87533e3027b9cc815be6ef81

[dist-8x8] Enable assert for 8 bit input only

Larger than 8 bit input seems to introduce different rounding methods in
different distortion calculation functions, which causes assertion failed
for dist-8x8's sanity check.

BUG=aomedia:1027

Change-Id: I1f607f32d4017d8f9abbbe0b09d9dbb01a63fe9c

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 852d32b..358760a 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2288,7 +2288,7 @@
                                 bw, bh, visible_w, visible_h, qindex);
 
   if (!is_inter_block(mbmi)) {
-    if (x->tune_metric == AOM_TUNE_PSNR) {
+    if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8) {
       assert(args->rd_stats.sse == tmp1 * 16);
       assert(args->rd_stats.dist == tmp2 * 16);
     }
@@ -2297,7 +2297,7 @@
   } else {
     // For inter mode, the decoded pixels are provided in pd->pred,
     // while the predicted pixels are in dst.
-    if (x->tune_metric == AOM_TUNE_PSNR) {
+    if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8) {
       assert(args->rd_stats.sse == tmp2 * 16);
       assert(args->rd_stats.dist == tmp1 * 16);
     }
@@ -4052,7 +4052,8 @@
       dist_8x8 = av1_dist_8x8(cpi, x, src, src_stride, dst, dst_stride,
                               BLOCK_8X8, 8, 8, 8, 8, qindex) *
                  16;
-      if (x->tune_metric == AOM_TUNE_PSNR) assert(sum_rd_stats.sse == dist_8x8);
+      if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
+        assert(sum_rd_stats.sse == dist_8x8);
       sum_rd_stats.sse = dist_8x8;
 
 #if CONFIG_HIGHBITDEPTH
@@ -4110,7 +4111,7 @@
       dist_8x8 = av1_dist_8x8(cpi, x, src, src_stride, pred8, 8, BLOCK_8X8, 8,
                               8, 8, 8, qindex) *
                  16;
-      if (x->tune_metric == AOM_TUNE_PSNR)
+      if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
         assert(sum_rd_stats.dist == dist_8x8);
       sum_rd_stats.dist = dist_8x8;
       tmp_rd = RDCOST(x->rdmult, sum_rd_stats.rate, sum_rd_stats.dist);