commit e4e18fcc661a8bf6cf7cd56d0d20fc7a3512a4e1
author Debargha Mukherjee <debargha@google.com> Wed Dec 06 23:43:24 2017 -0800
committer Debargha Mukherjee <debargha@google.com> Sat Dec 09 03:01:01 2017 +0000
tree a043a2686889cadab2c69623681adb9e4393f909
parent 4d8c6fc2e60ff47dc84776c2b8d4ac1403e5fb15

Enable 4:1 -> 2:1 -> 1:1 splits for 4:1 transforms

Also, splits the sub_tx_size_map array into inter and intra
in order to enable the new 4:1 transforms for inter and
intra separately.
Includes refactoring such as removing the intra_tx_size_cat_lookup
array since it is unnecessary, and consolidating the
max_txsize_rect_lookup array for convenience.

Change-Id: I112553bab612dafb973611c87f36a43e1ac4be85

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 1b75dc9..bfdf4be 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -230,7 +230,7 @@
                           xd->left_txfm_context + blk_row, tx_size, tx_size);
     // TODO(yuec): set correct txfm partition update for qttx
   } else {
-    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+    const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
     const int bsw = tx_size_wide_unit[sub_txs];
     const int bsh = tx_size_high_unit[sub_txs];
 
@@ -262,9 +262,9 @@
   if (block_signals_txsize(bsize)) {
     const TX_SIZE tx_size = mbmi->tx_size;
     const int tx_size_ctx = get_tx_size_context(xd);
-    const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
     const int depth = tx_size_to_depth(tx_size, bsize, 0);
     const int max_depths = bsize_to_max_depth(bsize, 0);
+    const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize, 0);
 
     assert(depth >= 0 && depth <= max_depths);
     assert(!is_inter_block(mbmi));
@@ -536,7 +536,7 @@
     token_stats->cost += tmp_token_stats.cost;
 #endif
   } else {
-    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+    const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
     const int bsw = tx_size_wide_unit[sub_txs];
     const int bsh = tx_size_high_unit[sub_txs];
 
@@ -592,7 +592,7 @@
     token_stats->cost += tmp_token_stats.cost;
 #endif
   } else {
-    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+    const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
     const int bsw = tx_size_wide_unit[sub_txs];
     const int bsh = tx_size_high_unit[sub_txs];
 
@@ -1111,7 +1111,7 @@
   const TX_SIZE mtx_size =
       get_max_rect_tx_size(xd->mi[0]->mbmi.sb_type, is_inter);
   const TX_SIZE tx_size =
-      is_inter ? AOMMAX(sub_tx_size_map[mtx_size], mbmi->min_tx_size)
+      is_inter ? AOMMAX(sub_tx_size_map[1][mtx_size], mbmi->min_tx_size)
                : mbmi->tx_size;
 #endif  // !CONFIG_TXK_SEL
   FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
@@ -1963,7 +1963,7 @@
     const int is_split =
         (l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
          txsize_to_bsize[l_max_tx_size] == bsizec);
-    if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
+    if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
   }
 #endif  // DISABLE_VARTX_FOR_CHROMA == 2
   const int step =