commit 94d5bfccdd5dbd0b04d41592f41031d6cc561aab
author Jingning Han <jingning@google.com> Fri Oct 21 10:14:36 2016 -0700
committer Jingning Han <jingning@google.com> Fri Oct 21 15:44:34 2016 -0700
tree 7f0f1e87d6a568ed1fc02538206566977d8ab9b6
parent 9777afc392504c77b241b4fcc4fc09f1eca14ce6

Limit the transform block partition depth

Limit the recursive transform block partition depth to 2. For a
32x32 transform block unit, one can maximally go down to 8x8 transform
block size.

Change-Id: I2caa92bb2eee64762b7ecca8920259f7c50fb0aa

av1/decoder/decodemv.c

diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 916832b..f0b37ce 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -257,8 +257,8 @@
 #if CONFIG_VAR_TX
 static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
                                MB_MODE_INFO *mbmi, FRAME_COUNTS *counts,
-                               TX_SIZE tx_size, int blk_row, int blk_col,
-                               aom_reader *r) {
+                               TX_SIZE tx_size, int depth, int blk_row,
+                               int blk_col, aom_reader *r) {
   int is_split = 0;
   const int tx_row = blk_row >> 1;
   const int tx_col = blk_col >> 1;
@@ -275,6 +275,19 @@
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
+  if (depth == 2) {
+    int idx, idy;
+    inter_tx_size[0][0] = tx_size;
+    for (idy = 0; idy < num_4x4_blocks_high_txsize_lookup[tx_size] / 2; ++idy)
+      for (idx = 0; idx < num_4x4_blocks_wide_txsize_lookup[tx_size] / 2; ++idx)
+        inter_tx_size[idy][idx] = tx_size;
+    mbmi->tx_size = tx_size;
+    if (counts) ++counts->txfm_partition[ctx][0];
+    txfm_partition_update(xd->above_txfm_context + tx_col,
+                          xd->left_txfm_context + tx_row, tx_size);
+    return;
+  }
+
   is_split = aom_read(r, cm->fc->txfm_partition_prob[ctx], ACCT_STR);
 
   if (is_split) {
@@ -297,8 +310,8 @@
     for (i = 0; i < 4; ++i) {
       int offsetr = blk_row + ((i >> 1) << bsl);
       int offsetc = blk_col + ((i & 0x01) << bsl);
-      read_tx_size_vartx(cm, xd, mbmi, counts, tx_size - 1, offsetr, offsetc,
-                         r);
+      read_tx_size_vartx(cm, xd, mbmi, counts, tx_size - 1, depth + 1, offsetr,
+                         offsetc, r);
     }
   } else {
     int idx, idy;
@@ -1755,8 +1768,8 @@
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
         for (idy = 0; idy < height; idy += bs)
           for (idx = 0; idx < width; idx += bs)
-            read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx,
-                               r);
+            read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size,
+                               height != width, idy, idx, r);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
       }
 #endif