commit b0a71306564187f07e578d829c512feed1eb9a83
author Jingning Han <jingning@google.com> Tue Oct 25 16:28:49 2016 -0700
committer Jingning Han <jingning@google.com> Wed Oct 26 17:12:41 2016 +0000
tree 008b784a666987582a7cd983f3c83da5b642999b
parent 8e022edd59bbfac8e54dc5e080179cef2bd3efcf

Convert tx_size to relative depth to fetch tx_size_cost

Use the relative transform partition depth as index to fetch the
tx_size_cost value.

Change-Id: I7d5119817baa96f23c32828065ff3175bb9f75cf

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 86db87c..f24c283 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1318,9 +1318,9 @@
   const int tx_size_cat =
       is_inter ? inter_tx_size_cat_lookup[bs] : intra_tx_size_cat_lookup[bs];
   const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
+  const int depth = tx_size_to_depth(coded_tx_size);
   const int tx_select = cm->tx_mode == TX_MODE_SELECT;
-  const int r_tx_size =
-      cpi->tx_size_cost[tx_size_cat][tx_size_ctx][coded_tx_size];
+  const int r_tx_size = cpi->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
 
   assert(skip_prob > 0);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
@@ -2746,7 +2746,7 @@
       // not the tokenonly rate.
       this_rate_tokenonly -=
           cpi->tx_size_cost[max_tx_size - TX_8X8][get_tx_size_context(xd)]
-                           [mic->mbmi.tx_size];
+                           [tx_size_to_depth(mic->mbmi.tx_size)];
     }
 #if CONFIG_PALETTE
     if (cpi->common.allow_screen_content_tools && mic->mbmi.mode == DC_PRED)
@@ -8014,7 +8014,7 @@
     // (prediction granularity), so we account for it in the full rate,
     // not the tokenonly rate.
     rate_y -= cpi->tx_size_cost[max_tx_size - TX_8X8][get_tx_size_context(xd)]
-                               [mbmi->tx_size];
+                               [tx_size_to_depth(mbmi->tx_size)];
   }
 
   rate2 += av1_cost_bit(cm->fc->filter_intra_probs[0],
@@ -8771,8 +8771,9 @@
         // tokenonly rate, but for intra blocks, tx_size is always coded
         // (prediction granularity), so we account for it in the full rate,
         // not the tokenonly rate.
-        rate_y -= cpi->tx_size_cost[max_tx_size - TX_8X8]
-                                   [get_tx_size_context(xd)][mbmi->tx_size];
+        rate_y -=
+            cpi->tx_size_cost[max_tx_size - TX_8X8][get_tx_size_context(xd)]
+                             [tx_size_to_depth(mbmi->tx_size)];
       }
 #if CONFIG_EXT_INTRA
       if (is_directional_mode) {