commit 5577bd18959d90dc1bcf4c54cd84572327616baf
author Debargha Mukherjee <debargha@google.com> Mon Nov 20 16:04:26 2017 -0800
committer Debargha Mukherjee <debargha@google.com> Tue Nov 21 17:48:50 2017 +0000
tree 79d508b77ffc0b7df011b27027a73abc9e9a0b63
parent 51021541fe8470d93f0fa9a7a8475488677765cb

Various RD fixes related to 4:1 transforms

The fixes in rdopt.c improves the coding performance of
4:1 transforms significantly.

Change-Id: I0e8db93e3f6d9bf0b2de01f2ce83c305d78d2262

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index bf552a4..7d26aa4 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1095,10 +1095,10 @@
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
   const int is_inter = is_inter_block(mbmi);
 #if !CONFIG_TXK_SEL
-  const TX_SIZE sqr_up_tx_size =
-      txsize_sqr_up_map[max_txsize_rect_lookup[xd->mi[0]->mbmi.sb_type]];
+  const TX_SIZE mtx_size =
+      get_max_rect_tx_size(xd->mi[0]->mbmi.sb_type, is_inter);
   const TX_SIZE tx_size =
-      is_inter ? AOMMAX(sub_tx_size_map[sqr_up_tx_size], mbmi->min_tx_size)
+      is_inter ? AOMMAX(sub_tx_size_map[mtx_size], mbmi->min_tx_size)
                : mbmi->tx_size;
 #endif  // !CONFIG_TXK_SEL
   FRAME_CONTEXT *ec_ctx = xd->tile_ctx;