commit 6001e04c6c5cfbeae20c54b06632bd590ff18b53
author Hui Su <huisu@google.com> Mon Jan 29 11:24:28 2018 -0800
committer Hui Su <huisu@google.com> Mon Jan 29 22:16:03 2018 +0000
tree f6d895e3534f278566614e6f8d9baeaeb4989a77
parent 6f1c2f7ccb391cfdb8e0f50d0377fae30b5c9103

Correctly handle transform type cost

Only consider transform type cost when there are more than just one
candidate.

Change-Id: I3b16bfb7f4d37a1826230396f46b741dbae82ac7

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 78525e5..d6e4371 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4898,15 +4898,13 @@
       if (this_rd_stats.rate != INT_MAX) {
         const int ext_tx_set = get_ext_tx_set(
             mbmi->min_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
-        TX_SIZE square_tx_size = txsize_sqr_map[mbmi->min_tx_size];
-#if CONFIG_TX64X64
-        // For TX_64X64, use TX_32X32 to lookup cost
-        if (square_tx_size == TX_64X64) square_tx_size = TX_32X32;
-#endif  // CONFIG_TX64X64
-        this_rd_stats.rate +=
-            x->inter_tx_type_costs[ext_tx_set][square_tx_size][mbmi->tx_type];
-        this_rd_stats.rate -=
-            x->inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type];
+        if (ext_tx_set > 0) {
+          const TX_SIZE square_tx_size = txsize_sqr_map[mbmi->min_tx_size];
+          this_rd_stats.rate +=
+              x->inter_tx_type_costs[ext_tx_set][square_tx_size][mbmi->tx_type];
+          this_rd_stats.rate -=
+              x->inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type];
+        }
       }
     }
 #endif  // CONFIG_TXK_SEL