commit c7ea761fe303bf8805396044bf601bf436a75018
author Jingning Han <jingning@google.com> Wed Jan 11 15:01:30 2017 -0800
committer Jingning Han <jingning@google.com> Thu Jan 12 00:40:03 2017 +0000
tree 84bcf1feadb45ceaf96a7b24df19e8d792b1a3ce
parent 0c70a80feb824acfa22691f4f3f3b040152c4c9d

Make rd_debug aligned to var-tx

Fix the corner case and use the right rate cost udpate for rd_debug.
This would make the var-tx pass rd_debug test.

Change-Id: Ib0fbd2d73030c0d150222c6b7c2dfffc0c6af085

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index ab4e47e..8bfb0e4 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -2946,7 +2946,7 @@
   for (tx_size = 0; tx_size <= max_tx_size; ++tx_size) {
     av1_coeff_stats frame_branch_ct[PLANE_TYPES];
     av1_coeff_probs_model frame_coef_probs[PLANE_TYPES];
-    if (cpi->td.counts->tx_size_totals[tx_size] <= 20 ||
+    if (cpi->td.counts->tx_size_totals[tx_size] <= 20 || CONFIG_RD_DEBUG ||
         (tx_size >= TX_16X16 && cpi->sf.tx_size_search_method == USE_TX_8X8)) {
       aom_write_bit(w, 0);
     } else {

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 0c7eea1..d0f0b99 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3344,6 +3344,10 @@
              RDCOST(x->rdmult, x->rddiv, zero_blk_rate, rd_stats->sse) ||
          rd_stats->skip == 1) &&
         !xd->lossless[mbmi->segment_id]) {
+#if CONFIG_RD_DEBUG
+      av1_update_txb_coeff_cost(rd_stats, plane, tx_size, blk_row, blk_col,
+                                zero_blk_rate - rd_stats->rate);
+#endif
       rd_stats->rate = zero_blk_rate;
       rd_stats->dist = rd_stats->sse;
       rd_stats->skip = 1;