commit cdef3b38141ac5e48ca866b4b4416437de779a86
author Hui Su <huisu@google.com> Wed Jan 17 12:36:31 2018 -0800
committer Hui Su <huisu@google.com> Wed Jan 17 23:49:58 2018 +0000
tree 677b094e952ec562118325b488075980db72ee75
parent 6f84e12d77e719ad698a7d2193b5d81cdb1668f3

Fix a bug in intra_block_yrd()

The tx size cost should be included in the total rate cost.

Change-Id: Ib8fe9cc67cf712ba1c7c8f539b390909c7e6be7b

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 6efe58d..caeef4d 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3185,7 +3185,7 @@
     // not the tokenonly rate.
     this_rate_tokenonly -= tx_size_cost(&cpi->common, x, bsize, mbmi->tx_size);
   }
-  int this_rate = this_rate_tokenonly + bmode_costs[mbmi->mode];
+  int this_rate = rd_stats.rate + bmode_costs[mbmi->mode];
   const int use_palette = mbmi->palette_mode_info.palette_size[0] > 0;
 #if CONFIG_FILTER_INTRA
   const int use_filter_intra = mbmi->filter_intra_mode_info.use_filter_intra;
@@ -3439,6 +3439,8 @@
   }
 #endif  // CONFIG_FILTER_INTRA
 
+  // If previous searches use only the default tx type, do an extra search for
+  // the best tx type.
   if (x->use_default_intra_tx_type) {
     *mbmi = best_mbmi;
     x->use_default_intra_tx_type = 0;