Various RD fixes related to 4:1 transforms

The fixes in rdopt.c improves the coding performance of
4:1 transforms significantly.

Change-Id: I0e8db93e3f6d9bf0b2de01f2ce83c305d78d2262
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 8b17582..1b23e77 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -871,10 +871,10 @@
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
   const int inter_block = is_inter_block(mbmi);
 #if !CONFIG_TXK_SEL
-  const TX_SIZE sqr_up_tx_size =
-      txsize_sqr_up_map[max_txsize_rect_lookup[xd->mi[0]->mbmi.sb_type]];
+  const TX_SIZE mtx_size =
+      get_max_rect_tx_size(xd->mi[0]->mbmi.sb_type, inter_block);
   const TX_SIZE tx_size =
-      inter_block ? AOMMAX(sub_tx_size_map[sqr_up_tx_size], mbmi->min_tx_size)
+      inter_block ? AOMMAX(sub_tx_size_map[mtx_size], mbmi->min_tx_size)
                   : mbmi->tx_size;
 #endif  // !CONFIG_TXK_SEL
   FRAME_CONTEXT *ec_ctx = xd->tile_ctx;