commit a1e48dccf2123b6b8bb37e9c444a2c1e5a006394
author Yue Chen <yuec@google.com> Mon Aug 29 17:29:33 2016 -0700
committer Yue Chen <yuec@google.com> Tue Sep 06 11:26:15 2016 -0700
tree d6480619523e9af705162413562eec90fa665c64
parent 5ebdf40d77f16757f6ec2020665d38a8821266cc

Make RECT_TX(>=8x8) work with VAR_TX

Bitstream syntax:
For a rectangular inter block, 'rect_tx' flag is sent to indicate if
the biggest rect tx is used. If no, continue to decode regular
recursive tx partition.

Change-Id: I127e35cc619b65acb5e9a0717f399cdcdb73fbf0

av1/common/loopfilter.c

diff --git a/av1/common/loopfilter.c b/av1/common/loopfilter.c
index 7086be8..2147bb8 100644
--- a/av1/common/loopfilter.c
+++ b/av1/common/loopfilter.c
@@ -1246,12 +1246,18 @@
         tx_size_mask = 0;
 
 #if CONFIG_VAR_TX
-      if (is_inter_block(mbmi) && !mbmi->skip)
-        tx_size =
-            (plane->plane_type == PLANE_TYPE_UV)
-                ? uv_txsize_lookup[sb_type][mbmi->inter_tx_size
-                                                [blk_row][blk_col]][ss_x][ss_y]
-                : mbmi->inter_tx_size[blk_row][blk_col];
+      if (is_inter_block(mbmi) && !mbmi->skip) {
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+        TX_SIZE mb_tx_size = is_rect_tx(mbmi->tx_size)
+                                 ? mbmi->tx_size
+                                 : mbmi->inter_tx_size[blk_row][blk_col];
+#else
+        TX_SIZE mb_tx_size = mbmi->inter_tx_size[blk_row][blk_col];
+#endif
+        tx_size = (plane->plane_type == PLANE_TYPE_UV)
+                      ? uv_txsize_lookup[sb_type][mb_tx_size][ss_x][ss_y]
+                      : mb_tx_size;
+      }
 
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
       tx_size_r =