commit ffbff57d9da22524dd0e70bc637ada959ccc7ff1
author Rupert Swarbrick <rupert.swarbrick@argondesign.com> Tue Dec 12 11:27:46 2017 +0000
committer Rupert Swarbrick <rupert.swarbrick@argondesign.com> Tue Dec 12 18:04:54 2017 +0000
tree 676367ffcb8f986cf29322ea5e40753802a39767
parent 765e34e3d944c423fccfc89c1db5eafe1bcfbd2c

Fix calculation of tx_type cost for rectangular transforms

This patch fixes up av1_tx_type_cost to match the code in
av1_write_tx_type. Beforehand, we wrongly assumed a 32x16 block needed
to signal its transform size (with rect-tx-ext & rect-tx-ext-intra)
because we were passing 16x16 to get_ext_tx_types.

I've also changed av1_write_tx_type to use get_min_tx_size rather than
inlining its body. No functional change, but it's probably better to
use the same helper function both times.

Change-Id: Iff6ee0bff2d332d5270fe0219db88c95e0b051d0

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 20c3547..0ee408a 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2338,7 +2338,7 @@
                      TX_SIZE tx_size, TX_TYPE tx_type) {
   if (plane > 0) return 0;
 
-  tx_size = get_min_tx_size(tx_size);
+  const TX_SIZE square_tx_size = get_min_tx_size(tx_size);
 
   const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
   const int is_inter = is_inter_block(mbmi);
@@ -2348,8 +2348,7 @@
         get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
     if (is_inter) {
       if (ext_tx_set > 0)
-        return x
-            ->inter_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]][tx_type];
+        return x->inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type];
     } else {
       if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) {
 #if CONFIG_FILTER_INTRA
@@ -2359,11 +2358,11 @@
                                              .filter_intra_mode[0]];
         else
           intra_dir = mbmi->mode;
-        return x->intra_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]]
-                                     [intra_dir][tx_type];
+        return x->intra_tx_type_costs[ext_tx_set][square_tx_size][intra_dir]
+                                     [tx_type];
 #else
-        return x->intra_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]]
-                                     [mbmi->mode][tx_type];
+        return x->intra_tx_type_costs[ext_tx_set][square_tx_size][mbmi->mode]
+                                     [tx_type];
 #endif
       }
     }