commit a1e48dccf2123b6b8bb37e9c444a2c1e5a006394
author Yue Chen <yuec@google.com> Mon Aug 29 17:29:33 2016 -0700
committer Yue Chen <yuec@google.com> Tue Sep 06 11:26:15 2016 -0700
tree d6480619523e9af705162413562eec90fa665c64
parent 5ebdf40d77f16757f6ec2020665d38a8821266cc

Make RECT_TX(>=8x8) work with VAR_TX

Bitstream syntax:
For a rectangular inter block, 'rect_tx' flag is sent to indicate if
the biggest rect tx is used. If no, continue to decode regular
recursive tx partition.

Change-Id: I127e35cc619b65acb5e9a0717f399cdcdb73fbf0

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 5a21ebd..718bddb 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -791,7 +791,7 @@
 #endif
 
 #if CONFIG_VAR_TX
-  // Assert not magic number (uninitialised).
+  // Assert not magic number (uninitialized).
   assert(x->blk_skip[plane][(blk_row << bwl) + blk_col] != 234);
 
   if (x->blk_skip[plane][(blk_row << bwl) + blk_col] == 0) {
@@ -976,9 +976,6 @@
     int idx, idy;
     int block = 0;
     int step = num_4x4_blocks_txsize_lookup[max_tx_size];
-#if CONFIG_EXT_TX && CONFIG_RECT_TX
-    const TX_SIZE tx_size = plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
-#endif
     av1_get_entropy_contexts(bsize, TX_4X4, pd, ctx.ta[plane], ctx.tl[plane]);
 #else
     const struct macroblockd_plane *const pd = &xd->plane[plane];
@@ -991,7 +988,7 @@
 
 #if CONFIG_VAR_TX
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-    if (tx_size >= TX_SIZES) {
+    if (is_rect_tx(mbmi->tx_size)) {
       av1_foreach_transformed_block_in_plane(xd, bsize, plane, encode_block,
                                              &arg);
     } else {