commit 70e5f3f02b1437539fac3350a2ed0da9fdc4c7d7
author Jingning Han <jingning@google.com> Wed Nov 09 17:03:07 2016 -0800
committer Jingning Han <jingning@google.com> Thu Nov 10 17:55:16 2016 +0000
tree 8ff5986cf6fc3f1dd4d1a560f9af69edd04cb149
parent af6df17516fa820c8c4e2ebc166e9122c28450fa

Support rectangular tx_size in recursive txfm partition

This commit supports the recursive transform block partition to
start with both square and rectangular transform block sizes.

Change-Id: Idaf29d50cb1f4876b43e5ba82c2609708c6b1926

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 2f1c9a1..3d1c3af 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1280,7 +1280,7 @@
       !(is_inter && skip) && !xd->lossless[segment_id]) {
 #if CONFIG_VAR_TX
     if (is_inter) {  // This implies skip flag is 0.
-      const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
+      const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize];
       const int bh = tx_size_high_unit[max_tx_size];
       const int bw = tx_size_wide_unit[max_tx_size];
       const int width = num_4x4_blocks_wide_lookup[bsize];
@@ -1912,7 +1912,7 @@
       if (is_inter_block(mbmi))
 #endif
       {
-        const TX_SIZE max_tx_size = max_txsize_lookup[plane_bsize];
+        const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize];
         int block = 0;
         const int step =
             tx_size_wide_unit[max_tx_size] * tx_size_high_unit[max_tx_size];