commit a9336328d499a6d3d7d215d35889adebc8eacdfd
author Jingning Han <jingning@google.com> Wed Nov 02 15:45:07 2016 -0700
committer Jingning Han <jingning@google.com> Wed Nov 02 15:48:07 2016 -0700
tree dcd2dbcd1d03cae7bd4f7767aefa5b14548c7737
parent 46003149e564c2a7c71c2808982d15bd141ee59c

Make recursive txfm encoding process support rectangular tx_size

This commit makes the encoding process of the recursive transform
block partition support both rectangular and square transform block
sizes as the starting point. If the coding block size is rectangular,
it would allow the transform block size to start from the largest
rectangular transform size, and recursive parse to the selected
coding sizes.

Change-Id: I576628b9166565bada6a918f0a1e67849dfef4cd

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 3a33f35..8b62a22 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -875,7 +875,9 @@
   if (tx_size == plane_tx_size) {
     encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg);
   } else {
-    int bsl = block_size_wide[bsize] >> (tx_size_wide_log2[0] + 1);
+    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+    // This is the square transform block partition entry point.
+    int bsl = tx_size_wide_unit[sub_txs];
     int i;
     assert(bsl > 0);
 #if CONFIG_EXT_TX
@@ -885,7 +887,6 @@
     for (i = 0; i < 4; ++i) {
       const int offsetr = blk_row + ((i >> 1) * bsl);
       const int offsetc = blk_col + ((i & 0x01) * bsl);
-      const TX_SIZE sub_txs = tx_size - 1;
       int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
 
       if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;