commit 721868c629a832bb4ed74e91df405b657ad06b27
author Yushin Cho <ycho@mozilla.com> Mon Nov 14 16:04:33 2016 +0900
committer Yushin Cho <ycho@mozilla.com> Mon Nov 14 08:50:15 2016 +0000
tree b3d0bd6e1230e6146c424bb7a2f3676497abe2bb
parent 55711e6f1cc012b438e47055726ef5513b93293b

Fix block_rd_txfm() and dist_block() for pvq

1. block_rd_txfm() : av1_optimize_b() should not be called
if pvq is enabled.
Setting t_above and t_left is missing when pvq is enabled,
so added.

2. dist_block() : The nextgen2v has new feature of computing
distortion in pixel domain for speed level = 0, 1, where pvq
works incorrectly since it requres the blank destination
buffer (w/o adding predicted pixels) passed to
inverse trasnform function.

Change-Id: Ia0ee426e796781ee56b4503ea425d447cf88ed8c

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 34adc6a..b2d78f9 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1094,8 +1094,15 @@
       } else
 #endif  // CONFIG_AOM_HIGHBITDEPTH
       {
+#if !CONFIG_PVQ
         aom_convolve_copy(dst, dst_stride, recon, MAX_TX_SIZE, NULL, 0, NULL, 0,
                           bsw, bsh);
+#else
+        int i, j;
+
+        for (j = 0; j < bsh; j++)
+          for (i = 0; i < bsw; i++) recon[j * MAX_TX_SIZE + i] = 0;
+#endif
         inv_txfm_add(dqcoeff, recon, MAX_TX_SIZE, &inv_txfm_param);
       }
 
@@ -1216,8 +1223,10 @@
     av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
                     AV1_XFORM_QUANT_FP);
 #endif  // CONFIG_NEW_QUANT
+#if !CONFIG_PVQ
     if (x->plane[plane].eobs[block])
       av1_optimize_b(cm, x, plane, block, tx_size, coeff_ctx);
+#endif
     dist_block(args->cpi, x, plane, block, blk_row, blk_col, tx_size,
                &this_rd_stats.dist, &this_rd_stats.sse);
   }
@@ -1238,6 +1247,8 @@
   args->t_left[blk_row] = (x->plane[plane].eobs[block] > 0);
 #else
   this_rd_stats.rate = x->rate;
+  args->t_above[blk_col] = !x->pvq_skip[plane];
+  args->t_left[blk_row] = !x->pvq_skip[plane];
 #endif
   rd1 = RDCOST(x->rdmult, x->rddiv, this_rd_stats.rate, this_rd_stats.dist);
   rd2 = RDCOST(x->rdmult, x->rddiv, 0, this_rd_stats.sse);
@@ -3129,6 +3140,7 @@
                   AV1_XFORM_QUANT_FP);
 #endif  // CONFIG_NEW_QUANT
 
+  // TODO(yushin) : If PVQ is enabled, this should not be called.
   av1_optimize_b(cm, x, plane, block, tx_size, coeff_ctx);
 
 // TODO(any): Use dist_block to compute distortion
@@ -3174,6 +3186,7 @@
     inv_txfm_param.tx_size = tx_size;
     inv_txfm_param.eob = p->eobs[block];
     inv_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];
+// TODO(yushin) : If PVQ is enabled, rec_buffer needs be set as zeros.
 #if CONFIG_AOM_HIGHBITDEPTH
     if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
       inv_txfm_param.bd = xd->bd;